{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Blackjack-v0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23:15:42 [INFO] action_space: Discrete(2)\n",
      "23:15:42 [INFO] observation_space: Tuple(Discrete(32), Discrete(11), Discrete(2))\n",
      "23:15:42 [INFO] np_random: RandomState(MT19937)\n",
      "23:15:42 [INFO] natural: False\n",
      "23:15:42 [INFO] dealer: [10, 4]\n",
      "23:15:42 [INFO] player: [4, 7]\n",
      "23:15:42 [INFO] spec: EnvSpec(Blackjack-v0)\n",
      "23:15:42 [INFO] id: Blackjack-v0\n",
      "23:15:42 [INFO] entry_point: gym.envs.toy_text:BlackjackEnv\n",
      "23:15:42 [INFO] reward_threshold: None\n",
      "23:15:42 [INFO] nondeterministic: False\n",
      "23:15:42 [INFO] max_episode_steps: None\n",
      "23:15:42 [INFO] _kwargs: {}\n",
      "23:15:42 [INFO] _env_name: Blackjack\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"Blackjack-v0\")\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On-policy Monte Carlo\n",
    "\n",
    "Monte Carlo prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ob2state(observation):\n",
    "    return observation[0], observation[1], int(observation[2])\n",
    "\n",
    "def evaluate_action_monte_carlo(env, policy, episode_num=500000):\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # play an episode\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n, p=policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # end of episode\n",
    "        g = reward # return\n",
    "        for state, action in state_actions:\n",
    "            c[state][action] += 1.\n",
    "            q[state][action] += (g - q[state][action]) / c[state][action]\n",
    "    return q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy = np.zeros((22, 11, 2, 2))\n",
    "policy[20:, :, :, 0] = 1 # stand when >=20\n",
    "policy[:20, :, :, 1] = 1 # hit when <20\n",
    "\n",
    "q = evaluate_action_monte_carlo(env, policy) # action value\n",
    "v = (q * policy).sum(axis=-1) # state value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23:16:44 [DEBUG] observation = (11, 10, False)\n",
      "23:16:44 [DEBUG] player = [8, 3], dealer = [10, 9]\n",
      "23:16:44 [DEBUG] action = 0\n",
      "23:16:44 [DEBUG] observation = (11, 10, False), reward = -1.0, done = True\n",
      "23:16:44 [INFO] episode reward: -1.00\n"
     ]
    }
   ],
   "source": [
    "def play_policy(env, policy=None):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    logging.debug('observation = %s', observation)\n",
    "    while True:\n",
    "        logging.debug('player = %s, dealer = %s', env.player, env.dealer)\n",
    "        if policy is None:\n",
    "            action = env.action_space.sample()\n",
    "        else:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n, p=policy[state])\n",
    "        logging.debug('action = %s', action)\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        logging.debug('observation = %s, reward = %s, done = %s',\n",
    "                observation, reward, done)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if done:\n",
    "            break\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "episode_reward, elapsed_steps = play_policy(env)\n",
    "logging.info(\"episode reward: %.2f\", episode_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(data):\n",
    "    fig, axes = plt.subplots(1, 2, figsize=(9, 4))\n",
    "    titles = ['without ace', 'with ace']\n",
    "    have_aces = [0, 1]\n",
    "    extent = [12, 22, 1, 11]\n",
    "    for title, have_ace, axis in zip(titles, have_aces, axes):\n",
    "        dat = data[extent[0]:extent[1], extent[2]:extent[3], have_ace].T\n",
    "        axis.imshow(dat, extent=extent, origin='lower')\n",
    "        axis.set_xlabel('player sum')\n",
    "        axis.set_ylabel('dealer showing')\n",
    "        axis.set_title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeAklEQVR4nO3deZhkBXnv8e9vemYYZmAYZTEKKGrAqLkGyWgkuKISNS5JbhZz5V5FI1l84pLFqDEm5vHGqInRJM/VcC8RolyNCyaIJsElIHoVg4iIEqMhKAiy78hM9/R7/6gzpullprrnnDpdNd/P8/TTVaeq3/NWT53fvH3qVJ1UFZIkSaO2pu8GJEnS3skhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhZC+U5I4kD9rF7Vckecooe5I0mcwb7YpDyF6oqvarqssBkpyW5A2jWG+SFyT5zCjWJWl16CtvNB4cQiRJUi8cQiZEkpOSfGTO9W8mef+c61cmObq5XEl+MMnJwPOAVza7TD8yp+TRSS5JcmuSv02yYU6tFzf1b0pyVpL7NcuPaGqvnXPfc5P8UpKHAu8Ejm3WdcsuHsdlSW5PcnmSX553+3OSXJzktiT/nuRpzfIDkpya5Jok30nyhiRTK/6FSlrSKPNm3nofnORTSW5MckOSM5JsmXP74UnOTHJ9c5+/nHPbC5tsuTnJPyV5QGu/EK2YQ8jkOA94XJI1Se4LrAOOA2hej90PuGTuD1TVKcAZwJubXabPmnPzzwNPAx4IPAJ4QVPreOCNze33Bb4FvG93zVXVZcCvAJ9r1rVlibteBzwT2AycBPxZkmOadT8a+Bvgt4EtwOOBK5qfOx2YAX4QeCRwAvBLu+tL0oqMJG8WEQb5cz/gocDhwB80650CzmaQSUcAh9JkU5KfAl4D/AxwMHA+8N6VPXS1ae3u76JxUFWXJ7kdOBo4CvgnBn9d/BBwLHB+Vc0uo+SfV9XVAM1fLEc3y58H/HVVXdTc9mrg5iRHtPQ4Pjrn6nlJzgEeB1wEvKhZ98eb27/T9HAf4OnAlqr6HnBnkj8DTgb+qo2+JP2nEebN/PV+E/hmc/X6JG8Ffr+5/mgGw8lvV9VMs2znMWi/DLyx+WOIJH8EvCbJA6rqW8voUy1zCJks5wFPZLA34DzgFuAJDELhvGXW+u6cy3cx2Lhpvl+084aquiPJjQz+6vjOSpqeK8nTGYTKUQz21G0EvtLcfDjwsUV+7AEM/hK7JsnOZWuAK/e0H0lLGkXe3EOSQ4A/Z/CHyf4MtvObm5sPB741ZwCZ6wHA25P86dxyDHLLIaRHvhwzWXaGwuOay+cxCIUnsHQoLPc0ylcz2KABSLIJOJDBAHJns3jjnPv/wLDrSrIP8CHgT4D7NC/ZfIxBWMBgqHjwIj96JbANOKiqtjRfm6vq4cM+KEnLNoq8me+NTY1HVNVm4ETumQ/3n3tM2hxXAr88Jx+2VNW+VfX/9rAf7SGHkMlyHvAkYN+quorB655PYzAkfGmJn7kWWPI9/Iv4v8BJSY5uhoY/Ai6oqiuq6noGw8iJSaaSvJB7Dg3XAoclWb9E7fXAPsD1wEyzV+SEObef2qz7yc1r0Ycm+aGqugY4B/jTJJub2x6c5AnLeFySlmcUeTPf/sAdwC1JDmVwfNhOXwCuAf44yaYkG5Ic19z2TuDVSR4O3z+Q/ef2oA+1xCFkglTVvzHYQM9vrt8GXA58tqp2LPFjpwIPS3JLkr8bYh2fBH6PwR6LaxgMGc+dc5cXMwiGG4GHA3P/0vgU8FXgu0luWKT27cBLgfcz2MX634Cz5tz+BZqDVYFbGYTgzr0y/4PBEPO15mc/yODAWUkdGEXeLOL1wDEMtv+PAmfO6WcH8CwGLw99G7gK+IXmtg8DbwLel+Q24FIGx5GpZ6na071jkiRJy+eeEEmS1AuHEEmS1AuHEEmS1AuHEEmS1Iux+LCy9Ws31r7rDmi/cFcH5XZRt6vjh5f1oYbLqdtN2a7+zTo5QPvIbjavB224efd3WqYrr5zhxptms/t7jq/16zbVhg1b+m5jaNkxRm8a6CpLZ7vKp676bb/uzIO6yZEj972x9ZoryZGxGEL2XXcAxz7opPYLTy/2wXp7Ltum2y/a0cZYd9/dSV12LPUOvT1THf2b1UwHdf/qoPZrAu858v27v9MynfCMBe+YnjgbNmzhUY/8tdbrpqP/z6Zu72Db7Oj/9HSVpXd1lE/THWQ03eTpTX9+cOs1Ac7+L6e3XnMlOeLLMZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRedDSFJ/jrJdUkunbPs3kk+nuQbzfd7dbV+SePPHJEmW5d7Qk4DnjZv2auAT1bVkcAnm+uStJTTMEekidXZEFJVnwZumrf4OcDpzeXTgZ/qav2Sxp85Ik22tSNe332q6hqAqromySFL3THJycDJABvWH8DsxvWtN5OZjh5+0n7J7dOt1wTIvvt2UremO+p3aqqTunyvWi/5svt/ovWaAAdNbWq95toF/8+vaivKkX323cK2A9vPkQ03bG+9JsCO/Te0XnPN9h2t1+zUvvt0U3efdZ2Uzbr2677myI+1XhNWT46s2gNTq+qUqtpaVVvXrd3YdzuSxtDcHFm7T/uhK2nPjHoIuTbJfQGa79eNeP2Sxp85Ik2IUQ8hZwHPby4/H/j7Ea9f0vgzR6QJ0eVbdN8LfA54SJKrkrwI+GPgqUm+ATy1uS5JizJHpMnW2YGpVfWLS9z05K7WKWmymCPSZFu1B6ZKkqTJ5hAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ6sbbvBoY2237JTO9ovyhA0nrJWtfNP1Vmp7up28HvAKBmq5O6Xbj31B2d1N1RU63XLMbn97pSNRXu3tL+7w7Wd1AT1t3efj5N3bGt9ZoAOzbv00ndqds6KUu2d5N7rG3/+TWVDv7zA3ZU+3VXkiPuCZEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb3oZQhJ8ookX01yaZL3JtnQRx+Sxpc5Io2/kQ8hSQ4FXgpsraofBqaA5466D0njyxyRJkNfL8esBfZNshbYCFzdUx+Sxpc5Io25taNeYVV9J8mfAN8GvgecU1Xn7PYHx+nolan2m83MjtZrArAmnZStHbOd1GVtR0/ZmZnWS95d61qvCTBLR8+FMbKSHJnaXmz+1rbWe9mxYar1mgCZrdZrzuy/T+s1Adbesb2Tup1Z282/Wc22n3tTtP88AJjtqO5y9fFyzL2A5wAPBO4HbEpy4iL3OznJhUkunJ65a9RtSlrFVpIj27ffOeo2Je1GH/sXngL8R1VdX1XTwJnAj8+/U1WdUlVbq2rrurUbR96kpFVt2Tmyfv2mkTcpadf6GEK+DTwmycYkAZ4MXNZDH5LGlzkiTYCRDyFVdQHwQeAi4CtND6eMug9J48sckSbDyA9MBaiq3wd+v491S5oM5og0/sbpPSeSJGmCOIRIkqRe7PblmCQ/s8jiW4GvVNV17bckadKYI5IWM8wxIS8CjgX+ubn+RODzwFFJ/rCq3t1Rb5ImhzkiaYFhhpBZ4KFVdS1AkvsA7wB+DPg0YHhI2h1zRNICwxwTcsTO4GhcBxxVVTcB0920JWnCmCOSFhhmT8j5Sc4GPtBc/6/Ap5NsAm7pqjFJE8UckbTAMEPISxgExnFAgL8BPlRVBTypw94kTQ5zRNICux1CmpD4YPMlSctmjkhazG6PCUnyM0m+keTWJLcluT3JbaNoTtJkMEckLWaYl2PeDDyrqjw5lKSVMkckLTDMu2OuNTgk7SFzRNICw+wJuTDJ3wJ/B2zbubCqzuyqKUkTxxyRtMAwQ8hm4C7ghDnLCjA8JA3LHJG0wDDvjjlpFI1ImlzmiKTFLDmEJHllVb05yV8w+IvlHqrqpZ12JmnsmSOSdmVXe0J2HkR24Sga2ZUUZNuO9gvXgkxsx/RM+zVnZ9uvCdTd23Z/p9Vkens3dXe0//y6evperdcEuGv91a3XnF04H7Rl1eRIBWb2nWq97rq7OtjegWxvf5tPV5nXUT6lq3y6+dZOymb9+tZr/vv2Q1qvCfDYDd9oveZKcmTJIaSqPtJcPL+qLl9pU5L2XuaIpF0Z5sDU05IcCvwLg7Ndnl9VX+m2LUkTxhyRtMAwB6Y+Psl64FHAE4GPJtmvqu7ddXOSJoM5Imkxux1CkjwWeFzztQU4Gzi/27YkTRJzRNJihnk55jwGB5W9EfhYVXV0ZKCkCWaOSFpgmCHkQAan33488NIks8Dnqur3Ou1M0iQxRyQtMMwxIbckuRw4HDgM+HFgXdeNSZoc5oikxQxzTMi/A18HPgO8EzjJXamSlsMckbSYYV6OObKquvkkGkl7C3NE0gJrhrjP/ZJ8OMl1Sa5N8qEkh3XemaRJYo5IWmCYIeRdwFnA/YBDgY80yyRpWOaIpAWGGUIOrqp3VdVM83UacHDHfUmaLOaIpAWGGUJuSHJikqnm60Tgxq4bkzRRzBFJCwwzhLwQ+Hngu8A1wM82yyRpWOaIpAWG+ZyQbwPPHkEvkiaUOSJpMcN8TsjBwIuBI+bev6r8K0bSUMwRSYsZ5nNC/p7BiaY+Aezoth1JE8ockbTAMEPIxqr6nc47kTTJzBFJCwxzYOrZSZ7R5kqTbEnywST/muSyJMe2WV/SqmOOSFpgyT0hSW4HCgjwmiTbgOnmelXV5j1Y79uBf6yqn02yHti4B7UkrVLmiKRdWXIIqar9u1hhks0MTuf9gmY92wFPZCVNIHNE0q4M8+6Y44CLq+rO5gOGjgHe1rzlbiUeBFwPvCvJjwBfBF5WVXfOW+/JwMkAG9ZtHu6Fo+Warg6KQnZ0cJ6uLmoCzHbzO+hMungiQO1o/1jJG3fs13pNgNlq/9+s62fBasiRffbdMtj/0rLZqW6ek+ump1uvueb2u1uvCUBHv4NMz3RStw68Vzd1b7ql9Zp37NjQek2AHZ1v9cMZ5pnzDuCuZkN/JfAt4N17sM61DALoHVX1SOBO4FXz71RVp1TV1qraun7tpj1YnaRVoPccWbfeHJFWm2GGkJmqKuA5wNur6u3AnuxivQq4qqouaK5/kEGYSJpc5oikBYYZQm5P8mrgROCjSaaAdStdYVV9F7gyyUOaRU8GvrbSepLGgjkiaYFhhpBfALYBL2o2/EOBt+zhen8dOCPJJcDRwB/tYT1Jq5s5ImmBYc4d813grXOufxv4mz1ZaVVdDGzdkxqSxoc5Imkx3RzSLEmStBsOIZIkqRe7HEKSTCV5z6iakTR5zBFJS9nlEFJVO4CDm49ElqRlM0ckLWWYs+heAXw2yVkMPhAIgKp665I/IUn3dAXmiKR5hhlCrm6+1rBnHy4kae9ljkhaYJi36L4eIMmm+edlkKRhmCOSFrPbd8ckOTbJ14DLmus/kuR/dd6ZpIlhjkhazDBv0X0b8BPAjQBV9WUGp9CWpGG9DXNE0jxDfU5IVV05b1H75z2XNNHMEUnzDXNg6pVJfhyo5i12L6XZpSpJQzJHJC0wzJ6QXwFewuCEU1cxOFHUSzrsSdLkMUckLTDMu2NuAJ43gl4kTShzRNJilhxCkvwFUEvdXlUv7aQjSRPDHJG0K7vaE3LhyLqQNKnMEUlLWnIIqarTR9mIpMljjkjald0eE5LkYOB3gIcBG3Yur6rjO+xroTVDvZt4eaY6qAnUumHedLQ8meno3Yxr0k3dmZlOytb27Z3UzdRUJ3U1sCpypCCz7ZddM91BUaA62DZrw7rWawLs2NjNuQmn1refpQBrbrurk7p17y2t11yTK1qvCTBbS75KumIrqTjM/8JnMHgr3QOB1zM4EdW/rGBdkvZe5oikBYYZQg6sqlOB6ao6r6peCDym474kTRZzRNICw+zrmm6+X5PkJxmcCfOw7lqSNIHMEUkLDDOEvCHJAcBvAn8BbAZe0WlXkiaNOSJpgWE+rOzs5uKtwJO6bUfSJDJHJC1mt8eEJDkqySeTXNpcf0SS13bfmqRJYY5IWswwB6b+b+DVNK/pVtUlwHO7bErSxDFHJC0wzBCysaq+MG9ZNx8CIWlSmSOSFhhmCLkhyYNpPockyc8C13TalaRJY45IWmCYd8e8BDgF+KEk3wH+Azix064kTRpzRNICw7w75nLgKUk2AWuq6vbu25I0ScwRSYtZcghJ8htLLAegqt7aUU+SJoQ5ImlXdrUnZP/m+0OARwFnNdefBXy6y6YkTQxzRNKSlhxCqur1AEnOAY7Zufs0yR8AHxhJd5LGmjkiaVeGeXfM/YG550/fDhzRSTeSJpU5ImmBYd4d827gC0k+zODtdT8NnN5pV5ImjTkiaYFh3h3zP5P8A/C4ZtFJVfWlbtuSNEnMEUmLGWZPCFV1EXBRmytOMgVcCHynqp7ZZm1Jq485Imm+YY4J6crLgMt6XL+k8WeOSGOslyEkyWHATwL/p4/1Sxp/5og0/vraE/I24JXAbE/rlzT+3oY5Io21oY4JaVOSZwLXVdUXkzxxF/c7GTgZYMO6zTDbQc5UtV+zy7odyJpu5tBaM9VJ3c40n+A5DnYwPs+vrqwoR9YfwPpbtrXfy0w3M9Bdh21svea+V3+v9ZoAU3e2/3sFWHPrnZ3U7Wp7z23t/36nOtreV0uO9LEn5Djg2UmuAN4HHJ/kPfPvVFWnVNXWqtq6fu2mUfcoaXVbdo6sW2eOSKvNyIeQqnp1VR1WVUcAzwU+VVWeTVPS0MwRaTL0+e4YSZK0Fxv5MSFzVdW5wLl99iBpvJkj0vhyT4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSerF2r4bGNpsBzV3dFEUMrOj/aLT0+3XBKqqk7rMzHRSNlNTndStDvqdrm567eJZ29GzYFWpNWFm07rW6669s5ttc7+v3dh6ze33O6D1mgBTt36vk7p0tL3zvbs7KdtFnl47vbn1mrB6csQ9IZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcjH0KSHJ7kn5NcluSrSV426h4kjTdzRJoMa3tY5wzwm1V1UZL9gS8m+XhVfa2HXiSNJ3NEmgAj3xNSVddU1UXN5duBy4BDR92HpPFljkiToY89Id+X5AjgkcAFi9x2MnAywIap/Vlz6x3tNzA93X5NYPbOuzqp24mOfgdV1U3dbds6qduFt59/Qid1tx23rvWaN858tvWaozJ0jqzdzD5X3dp+Azfe3H5NgEMObL3k+ou+2XpNgB233dZJ3TWbNnVSd/bOOzupS9J6yQ994tjWawJsOKH97L9+5txl/0xvB6Ym2Q/4EPDyqlrwDK6qU6pqa1VtXT+17+gblLTqLStH1m4cfYOSdqmXISTJOgbBcUZVndlHD5LGmzkijb8+3h0T4FTgsqp666jXL2n8mSPSZOhjT8hxwH8Hjk9ycfP1jB76kDS+zBFpAoz8wNSq+gzQ/tE7kvYa5og0GfzEVEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1ItUVd897FaS24Gv993HkA4Cbui7iWUYp37HqVcYr34fUlX7991El8yRTo1Tv+PUK4xXv8vOkbVdddKyr1fV1r6bGEaSC8elVxivfsepVxivfpNc2HcPI2COdGSc+h2nXmG8+l1JjvhyjCRJ6oVDiCRJ6sW4DCGn9N3AMoxTrzBe/Y5TrzBe/Y5Trys1To9xnHqF8ep3nHqF8ep32b2OxYGpkiRp8ozLnhBJkjRhHEIkSVIvVt0QkuSvk1yX5NI5y96S5F+TXJLkw0m29Nji9y3W65zbfitJJTmoj94Ws1S/SX49ydeTfDXJm/vqb64lngdHJ/l8kouTXJjk0X32uFOSw5P8c5LLmt/hy5rl907y8STfaL7fq+9eYZf9rsrtbCXMke6YI93Ya3OkqlbVF/B44Bjg0jnLTgDWNpffBLyp7z6X6rVZfjjwT8C3gIP67nM3v9snAZ8A9mmuH9J3n7vo9Rzg6c3lZwDn9t1n08t9gWOay/sD/wY8DHgz8Kpm+atW0fN2qX5X5XbW4vNnVT4+c2TkvZoj3fa7rO1s1e0JqapPAzfNW3ZOVc00Vz8PHDbyxhaxWK+NPwNeCayqo36X6PdXgT+uqm3Nfa4beWOLWKLXAjY3lw8Arh5pU0uoqmuq6qLm8u3AZcChwHOA05u7nQ78VC8NzrNUv6t1O1sJc6Q75kg39tYcWXVDyBBeCPxD300sJcmzge9U1Zf77mVIRwGPS3JBkvOSPKrvhnbh5cBbklwJ/Anw6n7bWSjJEcAjgQuA+1TVNTDYYIFDemxtUfP6nWtVb2ctWNWPzxzp1MsxR1q1JzkyVkNIkt8FZoAz+u5lMUk2Ar8LvK7vXpZhLXAv4DHAbwPvT5J+W1rSrwKvqKrDgVcAp/bczz0k2Q/4EPDyqrqt7352Z6l+V/t2tqdW++MzRzpnjrRoT3NkbIaQJM8Hngk8r5oXm1ahBwMPBL6c5AoGu6EuSvIDvXa1a1cBZ9bAF4BZBidMWo2eD5zZXP4AsCoOKANIso7BhnhGVe3s8dok921uvy+wKnZRw5L9jst2tmJj8vjMkW6ZIy1pI0fGYghJ8jTgd4BnV9VdffezlKr6SlUdUlVHVNURDDbMY6rquz23tit/BxwPkOQoYD2r94yNVwNPaC4fD3yjx16+r/mL71Tgsqp665ybzmIQeDTf/37UvS1mqX7HZTtbqXF5fOZI58yRFrSWI30fYTv/C3gvcA0wzWDjexHwTeBK4OLm651997lUr/Nuv4LVdVT7Yr/b9cB7gEuBi4Dj++5zF70+Fvgi8GUGrz3+aN99Nr0+lsHBbpfMeY4+AzgQ+CSDkPskcO++e91Nv6tyO2vx+bMqH585MvJezZFu+13WdubHtkuSpF6MxcsxkiRp8jiESJKkXjiESJKkXjiESJKkXjiESJKkXjiEiCTnJtnadx+Sxpc5opVwCFHnkkz13YOk8WaOTCaHkL1EkiOS/GuS05NckuSDzTkq5t/vHUkuTPLVJK9vlj05yYfn3OepSc5sLp+Q5HNJLkrygeY8AiS5IsnrknwG+Ll56/i5JJcm+XKSTzfLXpDkL+fc5+wkT2wu35HkTUm+mOQTSR7d/NV1eXOiL0kjYI6obQ4he5eHAKdU1SOA24BfW+Q+v1tVW4FHAE9I8gjgU8BDkxzc3Ock4F1JDgJeCzylqo4BLgR+Y06tu6vqsVX1vnnreB3wE1X1I8AwG/8m4Nyq+lHgduANwFOBnwb+cIifl9Qec0StcQjZu1xZVZ9tLr+HwcfuzvfzSS4CvgQ8HHhYDT5W993AiUm2AMcyOD3zY4CHAZ9NcjGD8xo8YE6tv12ij88CpyV5MTDMLtbtwD82l78CnFdV083lI4b4eUntMUfUmrV9N6CRmv8Z/fe4nuSBwG8Bj6qqm5OcBmxobn4X8BHgbuADVTXTnMDo41X1i0us785Fm6j6lSQ/BvwkcHGSoxmc8nnuULxhzuXp+s/zC8wC25o6s0l8DkujZY6oNe4J2bvcP8mxzeVfBD4z7/bNDDb4W5PcB3j6zhuq6moGZ598LXBas/jzwHFJfhAgycbmDJq7lOTBVXVBVb2OwZk2D2dwkq6jk6xJcjir6PTaku7BHFFrnP72LpcBz0/yVwzOyPiOuTdW1ZeTfAn4KnA5g92dc50BHFxVX2vuf32SFwDvTbJPc5/XAv+2mz7ekuRIIAzOCvnlZvl/MNg1uvNMnJJWH3NErfEsunuJJEcAZ1fVD+9Bjb8EvlRVp7bWmKSxYY6obe4J0VCSfJHBLtbf7LsXSePJHNF87gmRJEm98MBUSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUi/8PhvRxy0iLrxUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(level=logging.INFO, # reset display level to avoid too much output\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23:16:45 [INFO] average episode reward = -0.32 ± 0.93\n"
     ]
    }
   ],
   "source": [
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_policy(env, policy)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13, 10, False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], shape=(0, 2, 2), dtype=float64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "policy[tuple(env.reset())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Monte Carlo update with exploring start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monte_carlo_with_exploring_start(env, episode_num=500000):\n",
    "    policy = np.zeros((22, 11, 2, 2))\n",
    "    policy[:, :, :, 1] = 1.\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # choose initial state randomly\n",
    "        state = (np.random.randint(12, 22),\n",
    "                 np.random.randint(1, 11),\n",
    "                 np.random.randint(2))\n",
    "        action = np.random.randint(2)\n",
    "        # play an episode\n",
    "        env.reset()\n",
    "        if state[2]: # has ace\n",
    "            env.player = [1, state[0] - 11]\n",
    "        else: # no ace\n",
    "            if state[0] == 21:\n",
    "                env.player = [10, 9, 2]\n",
    "            else:\n",
    "                env.player = [10, state[0] - 10]\n",
    "        env.dealer[0] = state[1]\n",
    "        state_actions = []\n",
    "        while True:\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # end of episode\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n, p=policy[state])\n",
    "        g = reward # return\n",
    "        for state, action in state_actions:\n",
    "            c[state][action] += 1.\n",
    "            q[state][action] += (g - q[state][action]) / c[state][action]\n",
    "            a = q[state].argmax()\n",
    "            policy[state] = 0.\n",
    "            policy[state][a] = 1.\n",
    "    return policy, q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy, q = monte_carlo_with_exploring_start(env)\n",
    "v = q.max(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaB0lEQVR4nO3debSkdX3n8feHbhYbQRRaQkNjGxS3HGxJa0TEBRfcNY4aHZlRNKIZT1BjXFCjMcdxD2r0jIYJCirjhqiIJqAoi45C2ma33YIoS7MqizCyfuePetpc7tZ1+96q363q9+uce27V89R9ns/t0/U53/vUU/WkqpAkSRq2rVoHkCRJWyaHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiEbIGS/C7JH8+y/uIkTxxmJknjyb7RbBxCtkBVdfequgggydFJ3jWM/SZ5aZLvDWNfkhaHVn2j0eAQIkmSmnAIGRNJDkny9Qn3f5HkixPuX5JkdXe7ktwvyaHAi4E3dodMvz5hk6uTnJfk+iRfSLLdhG29otv+b5KckGRFt3xVt+2lEx57apK/TPIg4BPAft2+rpvl91if5MYkFyV55aT1z05yTpIbkvxHkqd0y++R5KgkG5JcluRdSZZs9j+opBkNs28m7XevJN9Jcm2Sa5Icm2SnCetXJjk+ydXdYz42Yd3Lum75bZKTktxnwf5BtNkcQsbHacABSbZKshuwNbA/QPd67N2B8yb+QFUdCRwLvL87ZPrMCatfADwFuC+wD/DSblsHAu/p1u8G/Ar4/KbCVdV64FXAD7p97TTDQ68CngHsCBwCfCjJvt2+HwF8GngDsBPwGODi7ueOAW4H7gc8DHgy8JebyiVpswylb6YRev2zAngQsBL4+26/S4AT6XXSKmB3um5K8hzgLcBzgeXAGcDnNu9X10JauumHaBRU1UVJbgRWA3sDJ9H76+KBwH7AGVV15xw2+U9VdTlA9xfL6m75i4FPVtW6bt3hwG+TrFqg3+MbE+6eluRk4ABgHfDybt/f6tZf1mXYFXgqsFNV/T/gpiQfAg4F/nkhckn6T0Psm8n7/QXwi+7u1UmOAN7R3X8EveHkDVV1e7ds4zlorwTe0/0xRJJ3A29Jcp+q+tUccmqBOYSMl9OAx9E7GnAacB3wWHqlcNoct3XFhNs303ty031ft3FFVf0uybX0/uq4bHNCT5TkqfRKZW96R+qWAed3q1cC35zmx+5D7y+xDUk2LtsKuGS+eSTNaBh9cxdJ7g38E70/THag9zz/bbd6JfCrCQPIRPcBPpLkHydujl5vOYQ05Msx42VjKRzQ3T6NXik8lplLYa6XUb6c3hMagCTbAzvTG0Bu6hYvm/D4P+p3X0m2Bb4MfBDYtXvJ5pv0ygJ6Q8Ve0/zoJcAtwC5VtVP3tWNVPaTfX0rSnA2jbyZ7T7eNfapqR+Bg7toPe048J22CS4BXTuiHnarqblX1f+eZR/PkEDJeTgMeD9ytqi6l97rnU+gNCWfP8DNXAjO+h38a/wc4JMnqbmh4N3BmVV1cVVfTG0YOTrIkycu469BwJbBHkm1m2PY2wLbA1cDt3VGRJ09Yf1S37yd0r0XvnuSBVbUBOBn4xyQ7duv2SvLYOfxekuZmGH0z2Q7A74DrkuxO7/ywjc4CNgDvTbJ9ku2S7N+t+wRweJKHwB9OZH/+PHJogTiEjJGq+hm9J+gZ3f0bgIuA71fVHTP82FHAg5Ncl+SrfezjFODv6B2x2EBvyHjhhIe8gl4xXAs8BJj4l8Z3gAuBK5JcM822bwQOA75I7xDrfwVOmLD+LLqTVYHr6ZXgxqMy/53eEPPj7mePo3firKQBGEbfTOOdwL70nv/fAI6fkOcO4Jn0Xh76NXAp8Bfduq8A7wM+n+QG4AJ655GpsVTN9+iYJEnS3HkkRJIkNeEQIkmSmnAIkSRJTTiESJKkJkbiw8p2udeSWrVy69YxNAc/O2/Zph+kReP33MStdUs2/cjRZY9o0Lb03tucHhmJIWTVyq0566SVrWNoDg5asbp1BM3BmXVK6wgDZ49o0Lb03tucHvHlGEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktTEwIaQJJ9MclWSCyYsu1eSbyX5eff9noPav6TRZ49I422QR0KOBp4yadmbgVOq6v7AKd19SZrJ0dgj0tga2BBSVacDv5m0+NnAMd3tY4DnDGr/kkafPSKNt2GfE7JrVW0A6L7fe6YHJjk0ydoka6++9o6hBZS06Nkj0phYtCemVtWRVbWmqtYs33lJ6ziSRpA9Ii1uwx5CrkyyG0D3/aoh71/S6LNHpDEx7CHkBOAl3e2XAF8b8v4ljT57RBoTg3yL7ueAHwAPSHJpkpcD7wWelOTnwJO6+5I0LXtEGm9LB7XhqnrRDKueMKh9Shov9og03hbtiamSJGm8OYRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmljaOoAkScN00IrVrSOo45EQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1ESTISTJ65JcmOSCJJ9Lsl2LHJJGlz0ijb6hDyFJdgcOA9ZU1Z8AS4AXDjuHpNFlj0jjodXLMUuBuyVZCiwDLm+UQ9LoskekETf0IaSqLgM+CPwa2ABcX1UnDzuHpNFlj0jjocXLMfcEng3cF1gBbJ/k4Gked2iStUnWXn3tHcOOKWkRs0ek8dDi5ZgnAr+sqqur6jbgeOBRkx9UVUdW1ZqqWrN85yVDDylpUbNHpDHQYgj5NfDIJMuSBHgCsL5BDkmjyx6RxkCLc0LOBI4D1gHndxmOHHYOSaPLHpHGw9IWO62qdwDvaLFvSePBHpFGn5+YKkmSmnAIkSRJTWzy5Zgkz51m8fXA+VV11cJHkjRu7BFJ0+nnnJCXA/sB3+3uPw74IbB3kn+oqs8MKJuk8WGPSJqinyHkTuBBVXUlQJJdgY8DfwacDlgekjbFHpE0RT/nhKzaWBydq4C9q+o3wG2DiSVpzNgjkqbo50jIGUlOBL7U3f8vwOlJtgeuG1QwSWPFHpE0RT9DyKvpFcb+QIBPA1+uqgIeP8BsksaHPSJpik0OIV1JHNd9SdKc2SOSprPJc0KSPDfJz5Ncn+SGJDcmuWEY4SSNB3tE0nT6eTnm/cAzq8qLQ0naXPaIpCn6eXfMlRaHpHmyRyRN0c+RkLVJvgB8Fbhl48KqOn5QoSSNHXtE0hT9DCE7AjcDT56wrADLQ1K/7BFJU/Tz7phDhhFE0viyRyRNZ8YhJMkbq+r9ST5K7y+Wu6iqwwaaTNLIs0ckzWa2IyEbTyJbO4wgs/nZecs4aMXq1jEkzd2i6RFJi8+MQ0hVfb27eUZVXTSkPJLGiD0iaTb9nJh6dJLdgX+nd7XLM6rq/MHGkjRm7BFJU/RzYupjkmwDPBx4HPCNJHevqnsNOpyk8WCPSJrOJoeQJI8GDui+dgJOBM4YbCxJ48QekTSdfl6OOY3eSWXvAb5ZVbcONpKkMWSPSJqinyFkZ3qX334McFiSO4EfVNXfDTSZpHFij0iaop9zQq5LchGwEtgDeBSw9aCDSRof9oik6fRzTsh/AD8Fvgd8AjjEQ6mS5sIekTSdfl6OuX9V3TnwJJLGmT0iaYqt+njMiiRfSXJVkiuTfDnJHgNPJmmc2COSpuhnCPkUcAKwAtgd+Hq3TJL6ZY9ImqKfIWR5VX2qqm7vvo4Glg84l6TxYo9ImqKfIeSaJAcnWdJ9HQxcO+hgksaKPSJpin6GkJcBLwCuADYAz+uWSVK/7BFJU/TzOSG/Bp41hCySxpQ9Imk6/XxOyHLgFcCqiY+vKv+KkdQXe0TSdPr5nJCv0bvQ1LeBOwYbR9KYskckTdHPELKsqt408CSSxpk9ImmKfk5MPTHJ0xZyp0l2SnJckp8kWZ9kv4XcvqRFxx6RNMWMR0KS3AgUEOAtSW4BbuvuV1XtOI/9fgT4t6p6XpJtgGXz2JakRcoekTSbGYeQqtphEDtMsiO9y3m/tNvPrYAXspLGkD0iaTabfDkmyf5Jtu9uH5zkiCR7zmOffwxcDXwqydlJ/mXj9ift99Aka5OsvY1b5rE7Sa0thh65+lrPh5UWm37OCfk4cHOShwJvBH4FfGYe+1wK7At8vKoeBtwEvHnyg6rqyKpaU1VrtmbbeexO0iLQvEeW77xkHruTNAj9DCG3V1UBzwY+UlUfAeZziPVS4NKqOrO7fxy9MpE0vuwRSVP0M4TcmORw4GDgG0mWAFtv7g6r6grgkiQP6BY9Afjx5m5P0kiwRyRN0c8Q8hfALcDLuyf+7sAH5rnfvwaOTXIesBp49zy3J2lxs0ckTdHPtWOuAI6YcP/XwKfns9OqOgdYM59tSBod9oik6fRzJESSJGnBOYRIkqQmZh1CkixJ8tlhhZE0fuwRSTOZdQipqjuA5d1HIkvSnNkjkmbSz1V0Lwa+n+QEeh8IBEBVHTHjT0jSXV2MPSJpkn6GkMu7r62Y34cLSdpy2SOSpujnLbrvBEiyfVXdtKnHS9Jk9oik6fRzAbv9kvwYWN/df2iS/zXwZJLGhj0iaTr9vEX3w8BBwLUAVXUuvUtoS1K/Pow9ImmSvj4npKoumbTIa2JLmhN7RNJk/ZyYekmSRwHVvcXuMLpDqpLUJ3tE0hT9HAl5FfBqehecupTehaJePcBMksaPPSJpin7eHXMN8OIhZJE0puwRSdOZcQhJ8lGgZlpfVYcNJJGksWGPSJrNbEdC1g4thaRxZY9ImtGMQ0hVHTPMIJLGjz0iaTabPCckyXLgTcCDge02Lq+qAweY6y723udmTjrpnAXf7kErVi/4NiVNtRh6RNLi08+7Y46l91a6+wLvpHchqn8fYCZJ48cekTRFP0PIzlV1FHBbVZ1WVS8DHjngXJLGiz0iaYp+Pqzstu77hiRPp3clzD0GF0nSGLJHJE3RzxDyriT3AF4PfBTYEXjdQFNJGjf2iKQp+vmwshO7m9cDjx9sHEnjyB6RNJ1NnhOSZO8kpyS5oLu/T5K3DT6apHFhj0iaTj8npv5v4HC613Sr6jzghYMMJWns2COSpuhnCFlWVWdNWnb7IMJIGlv2iKQp+hlCrkmyF931H5I8D9gw0FSSxo09ImmKft4d82rgSOCBSS4DfgkcPNBUksaNPSJpin7eHXMR8MQk2wNbVdWNg48laZzYI5KmM+MQkuRvZlgOQFUdMaBMksaEPSJpNrMdCdmh+/4A4OHACd39ZwKnDzKUpLFhj0ia0YxDSFW9EyDJycC+Gw+fJvl74EtDSSdppNkjkmbTz7tj9gRunXD/VmDVQNJIGlf2iKQp+nl3zGeAs5J8hd7b6/4cOGagqSSNG3tE0hT9vDvmfyb5V+CAbtEhVXX2YGNJGif2iKTp9HMkhKpaB6xbyB0nWQKsBS6rqmcs5LYlLT72iKTJ+jknZFBeA6xvuH9Jo88ekUZYkyEkyR7A04F/abF/SaPPHpFGX6sjIR8G3gjc2Wj/kkbfh7FHpJHW1zkhCynJM4CrqupHSR43y+MOBQ4F2I5lHLRi9VDySVr8NqdH9tx96HUnaRNaHAnZH3hWkouBzwMHJvns5AdV1ZFVtaaq1mzNtsPOKGlxm3OPLN95ybAzStqEoQ8hVXV4Ve1RVauAFwLfqSqvpimpb/aINB5avjtGkiRtwZq+SFpVpwKntswgabTZI9Lo8kiIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1MTS1gH6sfc+N3PSSecs+HYPWrF6wbcpSZL645EQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJamLoQ0iSlUm+m2R9kguTvGbYGSSNNntEGg9LG+zzduD1VbUuyQ7Aj5J8q6p+3CCLpNFkj0hjYOhHQqpqQ1Wt627fCKwHdh92Dkmjyx6RxkOLIyF/kGQV8DDgzGnWHQocCrDn7oOJedLl5wxku4KDVqxuHUFbiNY9otEzqO639+au2YmpSe4OfBl4bVXdMHl9VR1ZVWuqas3ynZcMP6CkRc8ekUZbkyEkydb0iuPYqjq+RQZJo80ekUZfi3fHBDgKWF9VRwx7/5JGnz0ijYcWR0L2B/4bcGCSc7qvpzXIIWl02SPSGBj6mVpV9T0gw96vpPFhj0jjwU9MlSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppIVbXOsElJbgR+2jpHn3YBrmkdYg5GKe8oZYXRyvuAqtqhdYhBskcGapTyjlJWGK28c+6RpYNKssB+WlVrWofoR5K1o5IVRivvKGWF0cqbZG3rDENgjwzIKOUdpawwWnk3p0d8OUaSJDXhECJJkpoYlSHkyNYB5mCUssJo5R2lrDBaeUcp6+Yapd9xlLLCaOUdpawwWnnnnHUkTkyVJEnjZ1SOhEiSpDHjECJJkppYdENIkk8muSrJBROWfSDJT5Kcl+QrSXZqGPEPpss6Yd3fJqkku7TINp2Z8ib56yQ/TXJhkve3yjfRDP8PVif5YZJzkqxN8oiWGTdKsjLJd5Os7/4NX9Mtv1eSbyX5eff9nq2zwqx5F+XzbHPYI4NjjwzGFtsjVbWovoDHAPsCF0xY9mRgaXf7fcD7WuecKWu3fCVwEvArYJfWOTfxb/t44NvAtt39e7fOOUvWk4GndrefBpzaOmeXZTdg3+72DsDPgAcD7wfe3C1/8yL6fztT3kX5PFvA/z+L8vezR4ae1R4ZbN45Pc8W3ZGQqjod+M2kZSdX1e3d3R8Ceww92DSmy9r5EPBGYFGd9TtD3r8C3ltVt3SPuWrowaYxQ9YCduxu3wO4fKihZlBVG6pqXXf7RmA9sDvwbOCY7mHHAM9pEnCSmfIu1ufZ5rBHBsceGYwttUcW3RDSh5cB/9o6xEySPAu4rKrObZ2lT3sDByQ5M8lpSR7eOtAsXgt8IMklwAeBw9vGmSrJKuBhwJnArlW1AXpPWODeDaNNa1LeiRb182wBLOrfzx4ZqNdijyyo+fTISA0hSd4K3A4c2zrLdJIsA94KvL11ljlYCtwTeCTwBuCLSdI20oz+CnhdVa0EXgcc1TjPXSS5O/Bl4LVVdUPrPJsyU97F/jybr8X++9kjA2ePLKD59sjIDCFJXgI8A3hxdS82LUJ7AfcFzk1yMb3DUOuS/FHTVLO7FDi+es4C7qR3waTF6CXA8d3tLwGL4oQygCRb03siHltVGzNemWS3bv1uwKI4RA0z5h2V59lmG5Hfzx4ZLHtkgSxEj4zEEJLkKcCbgGdV1c2t88ykqs6vqntX1aqqWkXviblvVV3RONpsvgocCJBkb2AbFu8VGy8HHtvdPhD4ecMsf9D9xXcUsL6qjpiw6gR6hUf3/WvDzjadmfKOyvNsc43K72ePDJw9sgAWrEdan2E7+Qv4HLABuI3ek+/lwC+AS4Bzuq9PtM45U9ZJ6y9mcZ3VPt2/7TbAZ4ELgHXAga1zzpL10cCPgHPpvfb4p61zdlkfTe9kt/Mm/B99GrAzcAq9kjsFuFfrrJvIuyifZwv4/2dR/n72yNCz2iODzTun55kf2y5JkpoYiZdjJEnS+HEIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQkeTUJGta55A0uuwRbQ6HEA1ckiWtM0gabfbIeHII2UIkWZXkJ0mOSXJekuO6a1RMftzHk6xNcmGSd3bLnpDkKxMe86Qkx3e3n5zkB0nWJflSdx0Bklyc5O1Jvgc8f9I+np/kgiTnJjm9W/bSJB+b8JgTkzyuu/27JO9L8qMk307yiO6vrou6C31JGgJ7RAvNIWTL8gDgyKraB7gB+B/TPOatVbUG2Ad4bJJ9gO8AD0qyvHvMIcCnkuwCvA14YlXtC6wF/mbCtn5fVY+uqs9P2sfbgYOq6qFAP0/+7YFTq+pPgRuBdwFPAv4c+Ic+fl7SwrFHtGAcQrYsl1TV97vbn6X3sbuTvSDJOuBs4CHAg6v3sbqfAQ5OshOwH73LMz8SeDDw/STn0LuuwX0mbOsLM+T4PnB0klcA/RxivRX4t+72+cBpVXVbd3tVHz8vaeHYI1owS1sH0FBN/oz+u9xPcl/gb4GHV9VvkxwNbNet/hTwdeD3wJeq6vbuAkbfqqoXzbC/m6YNUfWqJH8GPB04J8lqepd8njgUbzfh9m31n9cXuBO4pdvOnUn8PywNlz2iBeORkC3Lnkn2626/CPjepPU70nvCX59kV+CpG1dU1eX0rj75NuDobvEPgf2T3A8gybLuCpqzSrJXVZ1ZVW+nd6XNlfQu0rU6yVZJVrKILq8t6S7sES0Yp78ty3rgJUn+md4VGT8+cWVVnZvkbOBC4CJ6hzsnOhZYXlU/7h5/dZKXAp9Lsm33mLcBP9tEjg8kuT8QeleFPLdb/kt6h0Y3XolT0uJjj2jBeBXdLUSSVcCJVfUn89jGx4Czq+qoBQsmaWTYI1poHglRX5L8iN4h1te3ziJpNNkjmswjIZIkqQlPTJUkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1MT/B1m7NjtENEGGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeiElEQVR4nO3debhkB1nn8e+v7+0l3UmnExK2EAgiQZYHQ2yWsBMWAVmUcYEhM2wSdRhZXBAUUXwYEFAE9RkwTiABMqBA0LCoIEhYhsUkBAhEBGMgIZ2VLJ30fu87f9RpuLlLd93uc+rcqnw/z3OfrjpV9z3vrdT55a1Tp+qkqpAkSRq1VX03IEmSbpscQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQm6Dktyc5Mf2cfulSR43yp4kTSbzRvviEHIbVFWHVtUlAEnOSPLaUaw3yXOTfG4U65K0MvSVNxoPDiGSJKkXDiETIsnzknx4zvXvJPnbOdcvS3JCc7mS/HiSU4FnAy9vdpl+eE7JE5J8LcmNSf4mybo5tV7Y1P9BknOS3LlZflxTe3rOfT+d5JeT3Bt4O3BSs64b9vF3XJxka5JLkvzKvNufnuTCJDcl+Y8kT2yWH57k9CRbknw/yWuTTB3wAyppSaPMm3nrvUeSTyW5Lsm1Sc5KsmnO7ccmOTvJNc19/nLObc9vsuX6JP+U5G6tPSA6YA4hk+Nc4BFJViW5E7AaeBhA837socDX5v5CVZ0GnAW8sdll+tQ5N/8i8ETg7sD9gec2tU4GXt/cfifgu8D79tdcVV0M/CrwhWZdm5a469XAU4CNwPOAP0tyYrPuBwHvAn4b2AQ8Eri0+b0zgT3AjwMPAJ4A/PL++pJ0QEaSN4sIg/y5M3Bv4FjgD5v1TgEfYZBJxwHH0GRTkp8Ffhd4BnA08FngvQf2p6tN0/u/i8ZBVV2SZCtwAnA88E8MXl38BHAS8Nmqml1GyT+vqisAmlcsJzTLnw28o6ouaG57JXB9kuNa+js+OufquUk+DjwCuAB4QbPuTzS3f7/p4Q7Ak4BNVbUduCXJnwGnAn/VRl+SfmSEeTN/vd8BvtNcvSbJm4E/aK4/iMFw8ttVtadZtvcYtF8BXt+8GCLJ64DfTXK3qvruMvpUyxxCJsu5wKMZ7A04F7gBeBSDUDh3mbWunHN5G4ONm+bfC/beUFU3J7mOwauO7x9I03MleRKDUDmewZ669cDXm5uPBT62yK/djcErsS1J9i5bBVx2sP1IWtIo8uZWktwe+HMGL0wOY7CdX9/cfCzw3TkDyFx3A96a5E/nlmOQWw4hPfLtmMmyNxQe0Vw+l0EoPIqlQ2G5p1G+gsEGDUCSDcDtGAwgtzSL18+5/x2HXVeStcAHgT8B7tC8ZfMxBmEBg6HiHov86mXATuCoqtrU/GysqvsO+0dJWrZR5M18r29q3L+qNgKncOt8uOvcY9LmuAz4lTn5sKmqDqmq/3eQ/eggOYRMlnOBxwCHVNXlDN73fCKDIeErS/zOVcCSn+FfxP8FnpfkhGZoeB3wpaq6tKquYTCMnJJkKsnzufXQcBVwlyRrlqi9BlgLXAPsafaKPGHO7ac3635s8170MUl+oqq2AB8H/jTJxua2eyR51DL+LknLM4q8me8w4GbghiTHMDg+bK8vA1uAP06yIcm6JA9rbns78Mok94UfHsj+CwfRh1riEDJBqurfGWygn22u3wRcAny+qmaW+LXTgfskuSHJ3w2xjk8Cv89gj8UWBkPGM+fc5YUMguE64L7A3FcanwK+AVyZ5NpFam8FXgz8LYNdrP8VOGfO7V+mOVgVuJFBCO7dK/PfGQwx32x+9wMMDpyV1IFR5M0iXgOcyGD7/yhw9px+ZoCnMnh76HvA5cAvNbd9CHgD8L4kNwEXMTiOTD1L1cHuHZMkSVo+94RIkqReOIRIkqReOIRIkqReOIRIkqRejMWXla2ZXl+HrNnUfuGuDsqd7aDu7HK+fHA5OnoMprt5au1Z303d2dXt11y1cbHvTDp49zzkutZrXnbZHq77wWz2f8/xtXrNhlq37ojW63b2oHWQI+kim6C7fFrVzevkWtXNf7XZNe33O3PkUh80OjjHr7+m9ZqXXT7DD5aZI2MxhByyZhMPOf4FrdfNnm42nGzf2XrN2ra99ZoAzHTzBOfoIzspe/0Jt+uk7s3HtB8ehz72qtZrAnz4fu9qveYTnrzgE9MTZ926I9j8oP/Zet109GJmanv7Q+yqm3e1XhMgO7upW2uX+kqhgzN7aDd1t95t/f7vtEw3P+vG1msCfOzEv2695pMPIEd8O0aSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPWisyEkyTuSXJ3kojnLjkzyiSTfbv49oqv1Sxp/5og02brcE3IG8MR5y14BfLKq7gl8srkuSUs5A3NEmlidDSFV9RngB/MWPx04s7l8JvCzXa1f0vgzR6TJNj3i9d2hqrYAVNWWJLdf6o5JTgVOBVi7bhO7jtrQejOrt+5qvSZAptuf7XLI2tZrArBnppOytWbUT62Ds/3Eba3X/NN7ntN6TYCjptrfFqYX/H9+RTugHFmz4Qi23nVN681Mb6/WawJM72x/G8qeda3XBJja0U2OsCqdlJ1Z283r7ysf2n7N1937H9svCtxp+tDWa67O9cv+nRV7YGpVnVZVm6tq8+rV7YeupMk3N0em15kj0koz6iHkqiR3Amj+vXrE65c0/swRaUKMegg5B3hOc/k5wN+PeP2Sxp85Ik2ILj+i+17gC8C9klye5AXAHwOPT/Jt4PHNdUlalDkiTbbOjh6sqmctcdNju1qnpMlijkiTbcUemCpJkiabQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSerFdN8NDCUws7aLeWlNBzVhas1U6zWnb9jeek0AVnf0FJipTspee0I6qfuUe13Ues0Nq3a2XhNgZ820XnOWbv57rSS1CnZvaP/5Ux29lJvpIEdWb5ttvSbA7JpuciSz3Twvb7hHN/0+6sFfb73mpqltrdcEmKn2nwt1ADninhBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktSLXoaQJC9L8o0kFyV5b5J1ffQhaXyZI9L4G/kQkuQY4MXA5qq6HzAFPHPUfUgaX+aINBn6ejtmGjgkyTSwHriipz4kjS9zRBpz06NeYVV9P8mfAN8DtgMfr6qP7/N3psOuw6da72VqZzcz2JobOyi6sZs9zVPbdnVS96qTNnVSd+N9ruuk7uHT21uvuXX2kNZrAqxiW+s103rFbh1IjsxOw84j2+9lz45uHr2pHe3XrFXdZF5mOynL7kO7qbv9p9rfhrpy02w32b+ztrZesw7gd/p4O+YI4OnA3YE7AxuSnLLI/U5Ncl6S83bvuGXUbUpawQ4kR2a2mSPSStPH2zGPA/6zqq6pqt3A2cBD59+pqk6rqs1VtXn1ug0jb1LSirbsHJlab45IK00fQ8j3gIckWZ8kwGOBi3voQ9L4MkekCTDyIaSqvgR8ALgA+HrTw2mj7kPS+DJHpMkw8gNTAarqD4A/6GPdkiaDOSKNP78xVZIk9cIhRJIk9WK/b8ckecYii28Evl5VV7ffkqRJY45IWswwx4S8ADgJ+Jfm+qOBLwLHJ/mjqnp3R71JmhzmiKQFhhlCZoF7V9VVAEnuALwNeDDwGcDwkLQ/5oikBYY5JuS4vcHRuBo4vqp+AOzupi1JE8YckbTAMHtCPpvkI8D7m+v/BfhMkg3ADV01JmmimCOSFhhmCHkRg8B4GIPzXL0L+GBVFfCYDnuTNDnMEUkL7HcIaULiA82PJC2bOSJpMfs9JiTJM5J8O8mNSW5KsjXJTaNoTtJkMEckLWaYt2PeCDy1qjw5lKQDZY5IWmCYT8dcZXBIOkjmiKQFhtkTcl6SvwH+Dti5d2FVnd1VU5ImjjkiaYFhhpCNwDbgCXOWFWB4SBqWOSJpgWE+HfO8UTQiaXKZI5IWs+QQkuTlVfXGJH/B4BXLrVTVizvtTNLYM0ck7cu+9oTsPYjsvFE0si+zq2DXoWm97qp17dcE2Hn4mvaLVgc1gdnp9Z3U3XG7bh7b3Vu76ff86+/aes27rrmu9ZoAu9dtbb3mgumgPSsmR1gFM+va/0undnTzXN91ePs1d3eQowBTuzopS011U3fm2rWd1P3G4XdsveaDN17Sek2A3VzZes06gCRZcgipqg83Fz9bVd08CpImmjkiaV+GOTD1jCTHAP/K4GyXn62qr3fblqQJY45IWmCYA1MfmWQN8EDg0cBHkxxaVUd23ZykyWCOSFrMfoeQJA8HHtH8bAI+Any227YkTRJzRNJihnk75lwGB5W9HvhYVXV0CJKkCWaOSFpgmCHkdgxOv/1I4MVJZoEvVNXvd9qZpElijkhaYJhjQm5IcglwLHAX4KHA6q4bkzQ5zBFJixnmmJD/AL4FfA54O/A8d6VKWg5zRNJihnk75p5VNdt5J5ImmTkiaYFVQ9znzkk+lOTqJFcl+WCSu3TemaRJYo5IWmCYIeSdwDnAnYFjgA83yyRpWOaIpAWGGUKOrqp3VtWe5ucM4OiO+5I0WcwRSQsMM4Rcm+SUJFPNzylAN2fmkjSpzBFJCwwzhDwf+EXgSmAL8PPNMkkaljkiaYFhvifke8DTRtCLpAlljkhazDDfE3I08ELguLn3rypfxUgaijkiaTHDfE/I3zM40dQ/AzPdtiNpQpkjkhYYZghZX1W/03knkiaZOSJpgWEOTP1Ikie3udIkm5J8IMm/Jbk4yUlt1pe04pgjkhZYck9Ikq1AAQF+N8lOYHdzvapq40Gs963AP1bVzydZA6w/iFqSVihzRNK+LDmEVNVhXawwyUYGp/N+brOeXYAnspImkDkiaV+G+XTMw4ALq+qW5guGTgTe0nzk7kD8GHAN8M4kPwmcD7ykqm6Zt95TgVMB1qw/gtW31AGubml71qX1mgBTu9vvtTvdPAbT2zspy47r1nZS97oj2n8RvW22m1631e7Wa87S7XN2JeTI6sOO4JCr2n++7+lo/8v0tvZrzq5uvybA9LZunj+zq7vJpzU3DHMkwvLt2D3MYZbLc/muI1uvCbBttv3jw2dr+c+DYf5LvA3Y1mzoLwe+C7x72Wv6kWkGAfS2qnoAcAvwivl3qqrTqmpzVW2eXrfhIFYnaQXoP0cOMUeklWaYIWRPVRXwdOCtVfVW4GB2sV4OXF5VX2quf4BBmEiaXOaIpAWGGUK2JnklcArw0SRTwAHv1KuqK4HLktyrWfRY4JsHWk/SWDBHJC0wzBDyS8BO4AXNhn8M8KaDXO+vA2cl+RpwAvC6g6wnaWUzRyQtMMy5Y64E3jzn+veAdx3MSqvqQmDzwdSQND7MEUmL6eYQYUmSpP1wCJEkSb3Y5xCSZCrJe0bVjKTJY45IWso+h5CqmgGObr4SWZKWzRyRtJRhvt7tUuDzSc5h8IVAAFTVm5f8DUm6tUsxRyTNM8wQckXzs4qD+3IhSbdd5oikBYb5iO5rAJJsmH9eBkkahjkiaTH7/XRMkpOSfBO4uLn+k0n+d+edSZoY5oikxQzzEd23AD8NXAdQVV9lcAptSRrWWzBHJM0z1PeEVNVl8xa1fw5gSRPNHJE03zAHpl6W5KFANR+xezHNLlVJGpI5ImmBYfaE/CrwIgYnnLqcwYmiXtRhT5ImjzkiaYFhPh1zLfDsEfQiaUKZI5IWs+QQkuQvgFrq9qp6cScdSZoY5oikfdnXnpDzRtaFpElljkha0pJDSFWdOcpGJE0ec0TSvuz3mJAkRwO/A9wHWLd3eVWd3GFft+6hILPt111zy5J7iQ9KZtqvO7M2rdcEWL2rm8dg985u+q3V3fS7Ku3X3TbbzfnaOtgUln6/pCUrIUcqMDvM5wGXabqj739dtbv9/yqza7rZLqd2dlKWGupLJJYvMx09Dh3kyO6aar3mSjLMf+KzGHyU7u7AaxiciOpfO+xJ0uQxRyQtMMwQcruqOh3YXVXnVtXzgYd03JekyWKOSFpgmJ2Tu5t/tyT5GQZnwrxLdy1JmkDmiKQFhhlCXpvkcOA3gb8ANgIv67QrSZPGHJG0wDBfVvaR5uKNwGO6bUfSJDJHJC1mv8eEJDk+ySeTXNRcv3+SV3XfmqRJYY5IWswwB6b+NfBKmvd0q+prwDO7bErSxDFHJC0wzBCyvqq+PG/Zni6akTSxzBFJCwwzhFyb5B4032eU5OeBLZ12JWnSmCOSFhjm0zEvAk4DfiLJ94H/BE7ptCtJk8YckbTAMJ+OuQR4XJINwKqq2tp9W5ImiTkiaTFLDiFJfmOJ5QBU1Zs76knShDBHJO3LvvaEHNb8ey/ggcA5zfWnAp/psilJE8MckbSkJYeQqnoNQJKPAyfu3X2a5A+B94+kO0ljzRyRtC/DfDrmrsCuOdd3Acd10o2kSWWOSFpgmE/HvBv4cpIPMfh43c8BZ3balaRJY45IWmCYT8f8ryT/ADyiWfS8qvpKt21JmiTmiKTFDLMnhKq6ALigzRUnmQLOA75fVU9ps7aklccckTTfMMeEdOUlwMU9rl/S+DNHpDHWyxCS5C7AzwD/p4/1Sxp/5og0/vraE/IW4OXAbE/rlzT+3oI5Io21oY4JaVOSpwBXV9X5SR69j/udCpwKsPaQTUzvaD9nalVarzmo237N6e3VftEOZaabfldt62Zu3j0z1XrNmS6eCMDatF+3z/dlD8SB5Mjqw46ADjb5dDQC1VT7za7a1VGOdFR29S3dFN52x07KcvMt61qvuX1mdes1AWY6qHkg/7X6yJ6HAU9LcinwPuDkJO+Zf6eqOq2qNlfV5um1G0bdo6SVbdk5MrXeHJFWmpEPIVX1yqq6S1UdBzwT+FRVeTZNSUMzR6TJMG57YSVJ0oQY+TEhc1XVp4FP99mDpPFmjkjjyz0hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpF9N9NzCcUFNpveqq3dV6TQA6KDu9fab9osDs6m7m0Ont3dTNbCdl2TMzPvP4zmr/QejoYV1RMgvT29rfOLvIpkHh9kuuu6GbzNuzrqPHoCNTO7rpd6bar7s63WT/unTw/9QDqDk+yStJkiaKQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSerFyIeQJMcm+ZckFyf5RpKXjLoHSePNHJEmw3QP69wD/GZVXZDkMOD8JJ+oqm/20Iuk8WSOSBNg5HtCqmpLVV3QXN4KXAwcM+o+JI0vc0SaDH3sCfmhJMcBDwC+tMhtpwKnAqybOoyN51/R+vprx87WawLUzbe0X3Rmpv2awOyOHZ3UXTvdzVNr04Pv10ndax5wu9ZrvuPuj2m9JsDux021XvOaPee2XnNUhs6R1Ru547nXtb/+rdtarwnA7t2tl5y57vrWawKs2nhoJ3VnrvtBJ3WP2nR4J3W3P/ierdf8h80Pab0mAM9ov+SW3Z9Y9u/0dmBqkkOBDwIvraqb5t9eVadV1eaq2rxm6pDRNyhpxVtejqwffYOS9qmXISTJagbBcVZVnd1HD5LGmzkijb8+Ph0T4HTg4qp686jXL2n8mSPSZOhjT8jDgP8GnJzkwubnyT30IWl8mSPSBBj5galV9Tkgo16vpMlhjkiTwW9MlSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvUhV9d3DfiXZCnyr7z6GdBRwbd9NLMM49TtOvcJ49Xuvqjqs7ya6ZI50apz6HadeYbz6XXaOTHfVScu+VVWb+25iGEnOG5deYbz6HadeYbz6TXJe3z2MgDnSkXHqd5x6hfHq90ByxLdjJElSLxxCJElSL8ZlCDmt7waWYZx6hfHqd5x6hfHqd5x6PVDj9DeOU68wXv2OU68wXv0uu9exODBVkiRNnnHZEyJJkiaMQ4gkSerFihtCkrwjydVJLpqz7E1J/i3J15J8KMmmHlv8ocV6nXPbbyWpJEf10dtiluo3ya8n+VaSbyR5Y1/9zbXE8+CEJF9McmGS85I8qM8e90pybJJ/SXJx8xi+pFl+ZJJPJPl28+8RffcK++x3RW5nB8Ic6Y450o3bbI5U1Yr6AR4JnAhcNGfZE4Dp5vIbgDf03edSvTbLjwX+CfgucFTffe7nsX0M8M/A2ub67fvucx+9fhx4UnP5ycCn++6z6eVOwInN5cOAfwfuA7wReEWz/BUr6Hm7VL8rcjtr8fmzIv8+c2TkvZoj3fa7rO1sxe0JqarPAD+Yt+zjVbWnufpF4C4jb2wRi/Xa+DPg5cCKOup3iX5/DfjjqtrZ3OfqkTe2iCV6LWBjc/lw4IqRNrWEqtpSVRc0l7cCFwPHAE8Hzmzudibws700OM9S/a7U7exAmCPdMUe6cVvNkRU3hAzh+cA/9N3EUpI8Dfh+VX21716GdDzwiCRfSnJukgf23dA+vBR4U5LLgD8BXtlvOwslOQ54APAl4A5VtQUGGyxw+x5bW9S8fuda0dtZC1b032eOdOqlmCOtOpgcGashJMnvAXuAs/ruZTFJ1gO/B7y6716WYRo4AngI8NvA3yZJvy0t6deAl1XVscDLgNN77udWkhwKfBB4aVXd1Hc/+7NUvyt9OztYK/3vM0c6Z4606GBzZGyGkCTPAZ4CPLuaN5tWoHsAdwe+muRSBruhLkhyx1672rfLgbNr4MvALIMTJq1EzwHObi6/H1gRB5QBJFnNYEM8q6r29nhVkjs1t98JWBG7qGHJfsdlOztgY/L3mSPdMkda0kaOjMUQkuSJwO8AT6uqbX33s5Sq+npV3b6qjquq4xhsmCdW1ZU9t7YvfwecDJDkeGANK/eMjVcAj2ounwx8u8defqh5xXc6cHFVvXnOTecwCDyaf/9+1L0tZql+x2U7O1Dj8veZI50zR1rQWo70fYTt/B/gvcAWYDeDje8FwHeAy4ALm5+3993nUr3Ou/1SVtZR7Ys9tmuA9wAXARcAJ/fd5z56fThwPvBVBu89/lTffTa9PpzBwW5fm/McfTJwO+CTDELuk8CRffe6n35X5HbW4vNnRf595sjIezVHuu13WduZX9suSZJ6MRZvx0iSpMnjECJJknrhECJJknrhECJJknrhECJJknrhECKSfDrJ5r77kDS+zBEdCIcQdS7JVN89SBpv5shkcgi5jUhyXJJ/S3Jmkq8l+UBzjor593tbkvOSfCPJa5plj03yoTn3eXySs5vLT0jyhSQXJHl/cx4Bklya5NVJPgf8wrx1/EKSi5J8NclnmmXPTfKXc+7zkSSPbi7fnOQNSc5P8s9JHtS86rqkOdGXpBEwR9Q2h5DblnsBp1XV/YGbgP+xyH1+r6o2A/cHHpXk/sCngHsnObq5z/OAdyY5CngV8LiqOhE4D/iNObV2VNXDq+p989bxauCnq+ongWE2/g3Ap6vqp4CtwGuBxwM/B/zREL8vqT3miFrjEHLbcllVfb65/B4GX7s73y8muQD4CnBf4D41+FrddwOnJNkEnMTg9MwPAe4DfD7JhQzOa3C3ObX+Zok+Pg+ckeSFwDC7WHcB/9hc/jpwblXtbi4fN8TvS2qPOaLWTPfdgEZq/nf03+p6krsDvwU8sKquT3IGsK65+Z3Ah4EdwPurak9zAqNPVNWzlljfLYs2UfWrSR4M/AxwYZITGJzyee5QvG7O5d31o/MLzAI7mzqzSXwOS6Nljqg17gm5bblrkpOay88CPjfv9o0MNvgbk9wBeNLeG6rqCgZnn3wVcEaz+IvAw5L8OECS9c0ZNPcpyT2q6ktV9WoGZ9o8lsFJuk5IsirJsayg02tLuhVzRK1x+rttuRh4TpK/YnBGxrfNvbGqvprkK8A3gEsY7O6c6yzg6Kr6ZnP/a5I8F3hvkrXNfV4F/Pt++nhTknsCYXBWyK82y/+Twa7RvWfilLTymCNqjWfRvY1Ichzwkaq630HU+EvgK1V1emuNSRob5oja5p4QDSXJ+Qx2sf5m371IGk/miOZzT4gkSeqFB6ZKkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqRe/H/rd2G/L6tr6gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(policy.argmax(-1))\n",
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Monte Carlo update with soft poicy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monte_carlo_with_soft(env, episode_num=500000, epsilon=0.1):\n",
    "    policy = np.ones((22, 11, 2, 2)) * 0.5 # soft policy\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # play an episode\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n, p=policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # end of episode\n",
    "        g = reward # return\n",
    "        for state, action in state_actions:\n",
    "            c[state][action] += 1.\n",
    "            q[state][action] += (g - q[state][action]) / c[state][action]\n",
    "            # soft update\n",
    "            a = q[state].argmax()\n",
    "            policy[state] = epsilon / 2.\n",
    "            policy[state][a] += (1. - epsilon)\n",
    "    return policy, q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy, q = monte_carlo_with_soft(env)\n",
    "v = q.max(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaIElEQVR4nO3debSkdX3n8feHbhYbQRRaQkNrGxS3HGxJa0TEBRfcNY4aHZlRNKIZT1BjXFCjMcdxD2r0jIYZFFTGDVERTRpFWXQU0rTNZrsFUaCbVVnEyPqdP+ppc7lbV9++Vb9b1e/XOffcquep+zyfy+n68L1PPVVPqgpJkqRh2651AEmStG1yCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDyDYoyW+T/PEs6y9J8oRhZpI0nuwbzcYhZBtUVXetqosBkhyX5J3D2G+SlyT57jD2JWlhaNU3Gg0OIZIkqQmHkDGR5PAkX5tw/+dJvjDh/qVJVna3K8l9kxwBvAh4Q3fI9GsTNrkyyflJrk/y+SQ7TdjWy7vt/zrJyUmWdctXdNtePOGxpyf5yyQPBD4OHNjt67pZfo/1SW5McnGSV0xa/6wk65LckOTfkzy5W363JMcm2Zjk8iTvTLJozv9BJc1omH0zab/7Jvl2kmuTXJPkhCS7TVi/PMlJSa7uHvPRCete2nXLb5KsTnLvefsPojlzCBkfZwAHJ9kuyV7A9sBBAN3rsXcFzp/4A1V1DHAC8L7ukOkzJqx+PvBk4D7A/sBLum0dAry7W78X8Evgc5sLV1XrgVcC3+/2tdsMD70KeDqwK3A48MEkB3T7fjjwKeD1wG7Ao4FLup87HrgNuC/wUOBJwF9uLpekORlK30wj9PpnGfBAYDnw991+FwGn0OukFcDedN2U5NnAm4HnAEuBs4DPzu1X13xavPmHaBRU1cVJbgRWAvsBq+n9dfEA4EDgrKq6Yws2+U9VtQGg+4tlZbf8RcAnqmptt+4o4DdJVszT7/H1CXfPSHIqcDCwFnhZt+9vdusv7zLsCTwF2K2q/gO4KckHgSOAf56PXJL+0xD7ZvJ+fw78vLt7dZKjgbd39x9Obzh5fVXd1i3bdA7aK4B3d38MkeRdwJuT3LuqfrkFOTXPHELGyxnAY+kdDTgDuA54DL1SOGMLt3XFhNu/o/fkpvu+dtOKqvptkmvp/dVx+VxCT5TkKfRKZT96R+qWABd0q5cD35jmx+5N7y+xjUk2LdsOuHRr80ia0TD65k6S3BP4J3p/mOxC73n+m271cuCXEwaQie4NfDjJP07cHL3ecghpyJdjxsumUji4u30GvVJ4DDOXwpZeRnkDvSc0AEl2BnanN4Dc1C1eMuHxf9TvvpLsCHwJ+ACwZ/eSzTfolQX0hop9p/nRS4GbgT2qarfua9eqenC/v5SkLTaMvpns3d029q+qXYHDuHM/3GviOWkTXAq8YkI/7FZVd6mq/7eVebSVHELGyxnA44C7VNVl9F73fDK9IeGHM/zMlcCM7+Gfxv8FDk+yshsa3gWcXVWXVNXV9IaRw5IsSvJS7jw0XAnsk2SHGba9A7AjcDVwW3dU5EkT1h/b7fvx3WvReyd5QFVtBE4F/jHJrt26fZM8Zgt+L0lbZhh9M9kuwG+B65LsTe/8sE3OATYC70myc5KdkhzUrfs4cFSSB8MfTmR/3lbk0DxxCBkjVfVTek/Qs7r7NwAXA9+rqttn+LFjgQcluS7JV/rYx2nA39E7YrGR3pDxggkPeTm9YrgWeDAw8S+NbwMXAVckuWaabd8IHAl8gd4h1v8KnDxh/Tl0J6sC19MrwU1HZf47vSHmR93PnkjvxFlJAzCMvpnGO4AD6D3/vw6cNCHP7cAz6L089CvgMuAvunVfBt4LfC7JDcCF9M4jU2Op2tqjY5IkSVvOIyGSJKkJhxBJktSEQ4gkSWrCIUSSJDUxEh9Wtsc9FtWK5du3jjGWfnr+ks0/SGPv99zELXVzNv/I0WWPaBN7bzDm0iMjMYSsWL4956xe3jrGWDp02crWEbQAnF2ntY4wcPaINrH3BmMuPeLLMZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKmJgQ0hST6R5KokF05Ydo8k30zys+773Qe1f0mjzx6Rxtsgj4QcBzx50rI3AadV1f2A07r7kjST47BHpLE1sCGkqs4Efj1p8bOA47vbxwPPHtT+JY0+e0Qab8M+J2TPqtoI0H2/50wPTHJEkjVJ1lx97e1DCyhpwbNHpDGxYE9MrapjqmpVVa1auvui1nEkjSB7RFrYhj2EXJlkL4Du+1VD3r+k0WePSGNi2EPIycCLu9svBr465P1LGn32iDQmBvkW3c8C3wfun+SyJC8D3gM8McnPgCd29yVpWvaINN4WD2rDVfXCGVY9flD7lDRe7BFpvC3YE1MlSdJ4cwiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU00GUKSvDbJRUkuTPLZJDu1yCFpdNkj0ugb+hCSZG/gSGBVVf0JsAh4wbBzSBpd9og0Hlq9HLMYuEuSxcASYEOjHJJGlz0ijbjFw95hVV2e5APAr4D/AE6tqlOHnUM9qzesG8h2D122ciDblcAekcZFi5dj7g48C7gPsAzYOclh0zzuiCRrkqy5+trbhx1T0gJmj0jjocXLMU8AflFVV1fVrcBJwCMnP6iqjqmqVVW1aunui4YeUtKCZo9IY6DFEPIr4BFJliQJ8HhgfYMckkaXPSKNgaEPIVV1NnAisBa4oMtwzLBzSBpd9og0HoZ+YipAVb0deHuLfUsaD/aINPr8xFRJktSEQ4gkSWpisy/HJHnONIuvBy6oqqvmP5KkcWOPSJpOP+eEvAw4EPhOd/+xwA+A/ZL8Q1V9ekDZJI0Pe0TSFP0MIXcAD6yqKwGS7Al8DPgz4EzA8pC0OfaIpCn6OSdkxabi6FwF7FdVvwZuHUwsSWPGHpE0RT9HQs5Kcgrwxe7+fwHOTLIzcN2ggkkaK/aIpCn6GUJeRa8wDgICfAr4UlUV8LgBZpM0PuwRSVNsdgjpSuLE7kuStpg9Imk6mz0nJMlzkvwsyfVJbkhyY5IbhhFO0niwRyRNp5+XY94HPKOqvDiUpLmyRyRN0c+7Y660OCRtJXtE0hT9HAlZk+TzwFeAmzctrKqTBhVK0tixRyRN0c8QsivwO+BJE5YVYHlI6pc9ImmKft4dc/gwgkgaX/aIpOnMOIQkeUNVvS/JR+j9xXInVXXkQJNJGnn2iKTZzHYkZNNJZGuGEWScHLpsZesI0kJhj8zRIHpk9YZ1875NGFznDSqvFo4Zh5Cq+lp386yqunhIeSSNEXtE0mz6OTH1uCR7A/9G72qXZ1XVBYONJWnM2COSpujnxNRHJ9kBeBjwWODrSe5aVfcYdDhJ48EekTSdzQ4hSR4FHNx97QacApw12FiSxok9Imk6/bwccwa9k8reDXyjqm4ZbCRJY8gekTRFP0PI7vQuv/1o4MgkdwDfr6q/G2gySePEHpE0RT/nhFyX5GJgObAP8Ehg+0EHkzQ+7BFJ0+nnnJB/B34CfBf4OHC4h1IlbQl7RNJ0+nk55n5VdcfAk0gaZ/aIpCm26+Mxy5J8OclVSa5M8qUk+ww8maRxYo9ImqKfIeSTwMnAMmBv4GvdMknqlz0iaYp+hpClVfXJqrqt+zoOWDrgXJLGiz0iaYp+hpBrkhyWZFH3dRhw7aCDSRor9oikKfoZQl4KPB+4AtgIPLdbJkn9skckTdHP54T8CnjmELJIGlP2iKTp9PM5IUuBlwMrJj6+qvwrRlJf7BFJ0+nnc0K+Su9CU98Cbh9sHEljyh6RNEU/Q8iSqnrjwJNIGmf2iKQp+jkx9ZQkT53PnSbZLcmJSX6cZH2SA+dz+5IWHHtE0hQzHglJciNQQIA3J7kZuLW7X1W161bs98PAv1bVc5PsACzZim1JWqDsEUmzmXEIqapdBrHDJLvSu5z3S7r93AJ4IStpDNkjkmbTz7tjDgLWVdVN3QcMHQB8qHvL3Vz8MXA18MkkDwHOBV5dVTdN2u8RwBEAO7GEQ5etnOPuJLW2EHrkXnv3cwrcwrF6w7rWEfo2SllhcHn9/9SW6+eckI8Bv+ue6G8Afgl8eiv2uZheAX2sqh4K3AS8afKDquqYqlpVVau2Z8et2J2kBaB5jyzdfdFW7E7SIPQzhNxWVQU8C/hwVX0Y2JpDrJcBl1XV2d39E+mViaTxZY9ImqKfIeTGJEcBhwFfT7II2H6uO6yqK4BLk9y/W/R44Edz3Z6kkWCPSJqinyHkL4CbgZd1T/y9gfdv5X7/GjghyfnASuBdW7k9SQubPSJpin6uHXMFcPSE+78CPrU1O62qdcCqrdmGpNFhj0iaTj9HQiRJkuadQ4gkSWpi1iEkyaIknxlWGEnjxx6RNJNZh5Cquh1Y2n0ksiRtMXtE0kz6+QjBS4DvJTmZ3gcCAVBVR8/4E5J0Z5dgj0iapJ8hZEP3tR1b9+FCkrZd9oikKfp5i+47AJLsPPm6DJLUD3tE0nQ2++6YJAcm+RGwvrv/kCT/a+DJJI0Ne0TSdPp5i+6HgEOBawGq6jx6l9CWpH59CHtE0iR9fU5IVV06adHtA8giaYzZI5Im6+fE1EuTPBKo7i12R9IdUpWkPtkjkqbo50jIK4FX0bvg1GX0LhT1qgFmkjR+7BFJU/Tz7phrgBcNIYukMWWPSJrOjENIko8ANdP6qjpyIIkkjQ17RNJsZjsSsmZoKSSNK3tE0oxmHEKq6vhhBpE0fuwRSbPZ7DkhSZYCbwQeBOy0aXlVHTLAXJLGiD2iuTh02crWETRg/bw75gR6b6W7D/AOehei+rcBZpI0fuwRSVP0M4TsXlXHArdW1RlV9VLgEQPOJWm82COSpujnw8pu7b5vTPI0elfC3GdwkSSNIXtE0hT9DCHvTHI34HXAR4BdgdcONJWkcWOPSJqinw8rO6W7eT3wuMHGkTSO7BFJ09nsOSFJ9ktyWpILu/v7J3nr4KNJGhf2iKTp9HNi6v8GjqJ7TbeqzgdeMMhQksaOPSJpin6GkCVVdc6kZbcNIoyksWWPSJqinyHkmiT70l3/IclzgY0DTSVp3Ngjkqbo590xrwKOAR6Q5HLgF8BhA00ladzYI5Km6OfdMRcDT0iyM7BdVd04+FiSxok9Imk6Mw4hSf5mhuUAVNXRA8okaUzYI5JmM9uRkF267/cHHgac3N1/BnDmIENJGhv2iKQZzTiEVNU7AJKcChyw6fBpkr8HvjiUdJJGmj0iaTb9vDvmXsAtE+7fAqwYSBpJ48oekTRFP++O+TRwTpIv03t73Z8Dxw80laRxY49ImqKfd8f8zyT/AhzcLTq8qn442FiSxok9Imk6/RwJoarWAmvnc8dJFgFrgMur6unzuW1JC489Immyfs4JGZRXA+sb7l/S6LNHpBHWZAhJsg/wNOD/tNi/pNFnj0ijr9WRkA8BbwDuaLR/SaPvQ9gj0kjr65yQ+ZTk6cBVVXVuksfO8rgjgCMAdmLJcMJJGglz7ZFDl62c9yyrN6yb921K24oWR0IOAp6Z5BLgc8AhST4z+UFVdUxVraqqVduz47AzSlrY7BFpDAx9CKmqo6pqn6paAbwA+HZVeTVNSX2zR6Tx0PLdMZIkaRs29HNCJqqq04HTW2aQNNrsEWl0eSREkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJamJx6wD92G//37F69bp53+6hy1bO+zYlLUyD6hFJc+eREEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWpi6ENIkuVJvpNkfZKLkrx62BkkjTZ7RBoPixvs8zbgdVW1NskuwLlJvllVP2qQRdJoskekMTD0IyFVtbGq1na3bwTWA3sPO4ek0WWPSOOhxZGQP0iyAngocPY0644AjgC4196Dibl6w7qBbFdw6LKVrSNoG9G6RyTNXbMTU5PcFfgS8JqqumHy+qo6pqpWVdWqpbsvGn5ASQuePSKNtiZDSJLt6RXHCVV1UosMkkabPSKNvhbvjglwLLC+qo4e9v4ljT57RBoPLY6EHAT8N+CQJOu6r6c2yCFpdNkj0hgY+plaVfVdIMPer6TxYY9I48FPTJUkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqwiFEkiQ14RAiSZKaSFW1zrBZSW4EftI6R5/2AK5pHWILjFLeUcoKo5X3/lW1S+sQg2SPDNQo5R2lrDBaebe4RxYPKsk8+0lVrWodoh9J1oxKVhitvKOUFUYrb5I1rTMMgT0yIKOUd5SywmjlnUuP+HKMJElqwiFEkiQ1MSpDyDGtA2yBUcoKo5V3lLLCaOUdpaxzNUq/4yhlhdHKO0pZYbTybnHWkTgxVZIkjZ9RORIiSZLGjEOIJElqYsENIUk+keSqJBdOWPb+JD9Ocn6SLyfZrWHEP5gu64R1f5ukkuzRItt0Zsqb5K+T/CTJRUne1yrfRDP8O1iZ5AdJ1iVZk+ThLTNukmR5ku8kWd/9N3x1t/weSb6Z5Gfd97u3zgqz5l2Qz7O5sEcGxx4ZjG22R6pqQX0BjwYOAC6csOxJwOLu9nuB97bOOVPWbvlyYDXwS2CP1jk389/2ccC3gB27+/dsnXOWrKcCT+luPxU4vXXOLstewAHd7V2AnwIPAt4HvKlb/qYF9O92prwL8nk2j/9+FuTvZ48MPas9Mti8W/Q8W3BHQqrqTODXk5adWlW3dXd/AOwz9GDTmC5r54PAG4AFddbvDHn/CnhPVd3cPeaqoQebxgxZC9i1u303YMNQQ82gqjZW1dru9o3AemBv4FnA8d3Djgee3STgJDPlXajPs7mwRwbHHhmMbbVHFtwQ0oeXAv/SOsRMkjwTuLyqzmudpU/7AQcnOTvJGUke1jrQLF4DvD/JpcAHgKPaxpkqyQrgocDZwJ5VtRF6T1jgng2jTWtS3okW9PNsHizo388eGajXYI/Mq63pkZEaQpK8BbgNOKF1lukkWQK8BXhb6yxbYDFwd+ARwOuBLyRJ20gz+ivgtVW1HHgtcGzjPHeS5K7Al4DXVNUNrfNszkx5F/rzbGst9N/PHhk4e2QebW2PjMwQkuTFwNOBF1X3YtMCtC9wH+C8JJfQOwy1NskfNU01u8uAk6rnHOAOehdMWoheDJzU3f4isCBOKANIsj29J+IJVbUp45VJ9urW7wUsiEPUMGPeUXmezdmI/H72yGDZI/NkPnpkJIaQJE8G3gg8s6p+1zrPTKrqgqq6Z1WtqKoV9J6YB1TVFY2jzeYrwCEASfYDdmDhXrFxA/CY7vYhwM8aZvmD7i++Y4H1VXX0hFUn0ys8uu9fHXa26cyUd1SeZ3M1Kr+fPTJw9sg8mLceaX2G7eQv4LPARuBWek++lwE/By4F1nVfH2+dc6ask9ZfwsI6q326/7Y7AJ8BLgTWAoe0zjlL1kcB5wLn0Xvt8U9b5+yyPoreyW7nT/g3+lRgd+A0eiV3GnCP1lk3k3dBPs/m8d/Pgvz97JGhZ7VHBpt3i55nfmy7JElqYiRejpEkSePHIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQkSS05Osap1D0uiyRzQXDiEauCSLWmeQNNrskfHkELKNSLIiyY+THJ/k/CQndteomPy4jyVZk+SiJO/olj0+yZcnPOaJSU7qbj8pyfeTrE3yxe46AiS5JMnbknwXeN6kfTwvyYVJzktyZrfsJUk+OuExpyR5bHf7t0nem+TcJN9K8vDur66Luwt9SRoCe0TzzSFk23J/4Jiq2h+4Afgf0zzmLVW1CtgfeEyS/YFvAw9MsrR7zOHAJ5PsAbwVeEJVHQCsAf5mwrZ+X1WPqqrPTdrH24BDq+ohQD9P/p2B06vqT4EbgXcCTwT+HPiHPn5e0vyxRzRvHEK2LZdW1fe625+h97G7kz0/yVrgh8CDgQdV72N1Pw0clmQ34EB6l2d+BPAg4HtJ1tG7rsG9J2zr8zPk+B5wXJKXA/0cYr0F+Nfu9gXAGVV1a3d7RR8/L2n+2COaN4tbB9BQTf6M/jvdT3If4G+Bh1XVb5IcB+zUrf4k8DXg98AXq+q27gJG36yqF86wv5umDVH1yiR/BjwNWJdkJb1LPk8cineacPvW+s/rC9wB3Nxt544k/huWhsse0bzxSMi25V5JDuxuvxD47qT1u9J7wl+fZE/gKZtWVNUGeleffCtwXLf4B8BBSe4LkGRJdwXNWSXZt6rOrqq30bvS5nJ6F+lamWS7JMtZQJfXlnQn9ojmjdPftmU98OIk/0zviowfm7iyqs5L8kPgIuBieoc7JzoBWFpVP+oef3WSlwCfTbJj95i3Aj/dTI73J7kfEHpXhTyvW/4LeodGN12JU9LCY49o3ngV3W1EkhXAKVX1J1uxjY8CP6yqY+ctmKSRYY9ovnkkRH1Jci69Q6yva51F0miyRzSZR0IkSVITnpgqSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKmJ/w+cdlZgH5+0MwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAek0lEQVR4nO3debhkd13n8fenb++d7nQSAoYkEgwGEhgIsUEw7AEEZFFHFIfMsEnU4ZHFBQEZFB9GBBRBfAbMTCQBMqBAkBBQQZYQGBZDCJAQWYwhW2chC+l0SK/f+aNO481duuvePqfOrer363nuc6vOOfU937pV53u/9auzpKqQJEkatWV9JyBJkg5MNiGSJKkXNiGSJKkXNiGSJKkXNiGSJKkXNiGSJKkXNiEHoCS3J/mJvcy/IsnjR5mTpMlkvdHe2IQcgKrqoKq6HCDJmUleN4r1Jnluks+NYl2Sloa+6o3Gg02IJEnqhU3IhEjyvCQfmXb/u0n+btr9q5Kc2NyuJPdJchrwbODlzZDpR6aFPDHJ15P8IMnfJlk9LdYLm/g3Jzk3yT2b6cc0sZdPW/YzSX4tyfHAO4CHN+u6dS/P47IkW5JcnuTXZ8x/RpKLk9yW5N+SPKmZfnCSM5JsTnJNktclmVr0H1TSvEZZb2as99gkn0pyU5LvJzk7ycZp849Ock6SG5tl/mravOc3teWWJP+U5F6t/UG0aDYhk+N84JFJliU5AlgBnAzQfB97EPD16Q+oqtOBs4E3NkOmT5s2+5eBJwH3Bh4IPLeJ9Tjg9c38I4DvAe/bV3JVdRnwG8AXmnVtnGfRG4CnAhuA5wF/keSkZt0PBd4F/B6wEXgUcEXzuLOAncB9gAcDTwR+bV95SVqUkdSbOYRB/bkncDxwNPBHzXqngPMY1KRjgCNpalOSnwdeBfwicDhwAfDexT11tWn5vhfROKiqy5NsAU4EjgP+icGni/sBDwcuqKrdCwj5l1V1LUDzieXEZvqzgb+pqouaea8EbklyTEvP46PT7p6f5OPAI4GLgBc06/5EM/+aJod7AE8GNlbVD4GtSf4COA346zbykvQfRlhvZq73u8B3m7s3Jnkz8IfN/YcyaE5+r6p2NtP27IP268Drmw9DJPkT4FVJ7lVV31tAnmqZTchkOR94DIPRgPOBW4FHMygK5y8w1nXTbt/BYOOm+X3RnhlVdXuSmxh86rhmMUlPl+TJDIrKcQxG6tYC32hmHw18bI6H3YvBJ7HNSfZMWwZctb/5SJrXKOrNXSS5O/CXDD6YrGewnd/SzD4a+N60BmS6ewFvTfLn08MxqFs2IT3y65jJsqcoPLK5fT6DovBo5i8KC72M8rUMNmgAkqwDDmPQgGxtJq+dtvyPDbuuJKuADwJ/Btyj+crmYwyKBQyaimPneOhVwDbgblW1sfnZUFX3H/ZJSVqwUdSbmV7fxHhgVW0ATuWu9eHHp++TNs1VwK9Pqw8bq2pNVf2//cxH+8kmZLKcDzwWWFNVVzP43vNJDJqEr87zmOuBeY/hn8P/BZ6X5MSmafgT4EtVdUVV3cigGTk1yVSS53PXpuF64KgkK+eJvRJYBdwI7GxGRZ44bf4ZzbpPab6LPjLJ/apqM/Bx4M+TbGjmHZvk0Qt4XpIWZhT1Zqb1wO3ArUmOZLB/2B5fBjYDf5pkXZLVSU5u5r0DeGWS+8OPdmR/5n7koZbYhEyQqvo2gw30gub+bcDlwOeratc8DzsDOCHJrUn+foh1fBL4HwxGLDYzaDKeNW2RFzIoDDcB9wemf9L4FHApcF2S788RewvwYuDvGAyx/hfg3Gnzv0yzsyrwAwZFcM+ozH9j0MR8s3nsBxjsOCupA6OoN3N4LXASg+3/o8A50/LZBTyNwddDVwJXA7/SzPsQ8AbgfUluAy5hsB+Zepaq/R0dkyRJWjhHQiRJUi9sQiRJUi9sQiRJUi9sQiRJUi/G4mRlK5evrTUrN7YfuKudcncv5ESBw8bsagfijuIu7+attWtNR3FXtR9zav2O9oMC91lzc+sxr7pqJzfdvDv7XnJ8rVi1rlatO7TvNIbXwaaZzmpeN2HH7WPyzjXtb0LL1s917rX9d581N7Ue86qrdnLzAuvIWDQha1Zu5GHHvaD1uNnWzYubO7e1HrPuuLP1mIPAHVWPw7sp9rfdv5u4t96n/WvdHfrYza3HBPjwCWe3HvOUp9zYesylZtW6Q/lPT3xp63HT0QeEdLBpTt3ZzfbeVdxda8arC7nxgStaj7nhUde3HhPgww94V+sxf/Yps868sE/j9QpLkqSJYRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ60VkTkuRvktyQ5JJp0w5N8okk32l+H9LV+iWNP+uINNm6HAk5E3jSjGmvAD5ZVT8JfLK5L0nzORPriDSxOmtCquqzwM0zJj8DOKu5fRbw812tX9L4s45Ik235iNd3j6raDFBVm5Pcfb4Fk5wGnAawatVGdhy2tvVklm/Z3npMgGVTHfR2K1e0HxPI7Xd0Enfn+tWdxN2xppu+efuDtrYe81XHfqz1mACHTHWwLYzX7mGLqiMr1h/C1nu0/zyzu/WQAKz6QfuBa1k3r/PuFekk7q5V3eRbHb3df3i/O1uP+cfHfqL1mAB3n1rXeswVsz4v7NuSrTxVdXpVbaqqTStXtv/HkjT5pteR5WusI9JSM+om5PokRwA0v28Y8foljT/riDQhRt2EnAs8p7n9HODDI16/pPFnHZEmRJeH6L4X+AJw3yRXJ3kB8KfAE5J8B3hCc1+S5mQdkSZbZzumVtWvzjPrlK7WKWmyWEekybZkd0yVJEmTzSZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1YnnfCQyjlsHONVOtx83OFa3HBNi9sv1cl/1wZ+sxAaZ27e4k7u5V3by1br5/J2F55v2+2nrMjcvuaD0mwK5K6zGLaj3mgWLbId3Eran2PyMu/2E3r/Oy7Z2EZcf69t/rALcf3c3f4QnHX9Z6zKNX3NR6TIBd1f7/qcXUEUdCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL3ppQpK8LMmlSS5J8t4kq/vIQ9L4so5I42/kTUiSI4EXA5uq6gHAFPCsUechaXxZR6TJ0NfXMcuBNUmWA2uBa3vKQ9L4so5IY275qFdYVdck+TPgSuCHwMer6uN7fUzCrlUd9Evp5ukvv2NX6zGzc3frMQF2HbKuk7g3n9DNyPia+93aSdxlqdZjbq2Vrccc2NFR3PGxmDqyayXcfq/2t6OVt3TzWW7n2vZjbtuY9oMCq27tJCzbD+4m7hEnXtdJ3MNX3t56zGt3HtJ6TIAHrbyl9ZiLqaJ9fB1zCPAM4N7APYF1SU6dY7nTklyY5MId29p/YSWNr8XUkV1bt446TUn70MfXMY8H/r2qbqyqHcA5wM/MXKiqTq+qTVW1acWqg0aepKQlbcF1ZGpdN6N+khavjybkSuBhSdYmCXAKcFkPeUgaX9YRaQKMvAmpqi8BHwAuAr7R5HD6qPOQNL6sI9JkGPmOqQBV9YfAH/axbkmTwToijT/PmCpJknphEyJJknqxz69jkvziHJN/AHyjqm5oPyVJk8Y6Imkuw+wT8gLg4cCnm/uPAb4IHJfkj6vq3R3lJmlyWEckzTJME7IbOL6qrgdIcg/g7cBPA58FLB6S9sU6ImmWYfYJOWZP4WjcABxXVTfj+aMlDcc6ImmWYUZCLkhyHvD+5v5/Bj6bZB1wa1eJSZoo1hFJswzThLyIQcE4GQjwLuCDVVXAYzvMTdLksI5ImmWfTUhTJD7Q/EjSgllHJM1ln/uEJPnFJN9J8oMktyXZkuS2USQnaTJYRyTNZZivY94IPK2qvDiUpMWyjkiaZZijY663cEjaT9YRSbMMMxJyYZK/Bf4e2LZnYlWd01VSkiaOdUTSLMM0IRuAO4AnTptWgMVD0rCsI5JmGebomOeNIhFJk8s6Imku8zYhSV5eVW9M8jYGn1juoqpe3GlmksaedUTS3uxtJGTPTmQXjiKRvallsGNdOog8zH65C3f7EcN8y7Uwy3asaj0mwO4VnYTlzsO6eL1g29bVncT95m0/1nrMe6+6sfWYANtWXdl6zFndQXuWTB0BBqdJazvk7vZjAqzd3P6rcsvxrYcEYMMV3cS944hu4l5/6/pO4n573d1bj3nUyptbjwmwZff1+15ogXYvopLM+9+yqj7S3Lygqi5fbFKSDlzWEUl7M8xH9jOTHAn8C4OrXV5QVd/oNi1JE8Y6ImmWYXZMfVSSlcBDgMcAH01yUFUd2nVykiaDdUTSXPbZhCR5BPDI5mcjcB5wQbdpSZok1hFJcxnm65jzGexU9nrgY1W1vduUJE0g64ikWYZpQg5jcPntRwEvTrIb+EJV/Y9OM5M0SawjkmYZZp+QW5NcDhwNHAX8DNDRgZ2SJpF1RNJchtkn5N+AbwGfA94BPM+hVEkLYR2RNJdhvo75yarq6HQ8kg4Q1hFJswxzytB7JvlQkhuSXJ/kg0mO6jwzSZPEOiJplmGakHcC5wL3BI4EPtJMk6RhWUckzTJME3J4Vb2zqnY2P2cCh3ecl6TJYh2RNMswTcj3k5yaZKr5ORW4qevEJE0U64ikWYZpQp4P/DJwHbAZ+KVmmiQNyzoiaZZhzhNyJfD0EeQiaUJZRyTNZZjzhBwOvBA4ZvryVeWnGElDsY5Imssw5wn5MIMLTf0zsKvbdCRNKOuIpFmGaULWVtXvd56JpElmHZE0yzA7pp6X5CltrjTJxiQfSPKvSS5L8vA240tacqwjkmaZdyQkyRaggACvSrIN2NHcr6rasB/rfSvwj1X1S0lWAmv3I5akJco6Imlv5m1Cqmp9FytMsoHB5byf26xnO+CFrKQJZB2RtDfDHB1zMnBxVW1tTjB0EvCW5pC7xfgJ4EbgnUkeBHwFeElVbZ2x3tOA0wBWrjuEWpZFrm5+Pzys/ZgAGaPLdC3b0U3cqTu7ibtrSzdXf79lW/sfou/Yvar1mONqKdSR5RsOYeXNw3wDvTDb7tbRBp/2c53aXq3HBNhydEe1dGc3+e66cl0nca/buD8De3O7ecNBrccEmKKb12yhhnmXvx24o9nQXw58D3j3fqxzOYMC9PaqejCwFXjFzIWq6vSq2lRVm5av7uYNI2lkeq8jU+usI9JSM0wTsrOqCngG8NaqeiuwP0OsVwNXV9WXmvsfYFBMJE0u64ikWYZpQrYkeSVwKvDRJFPAosfEq+o64Kok920mnQJ8c7HxJI0F64ikWYZpQn4F2Aa8oNnwjwTetJ/r/S3g7CRfB04E/mQ/40la2qwjkmYZ5tox1wFvnnb/SuBd+7PSqroY2LQ/MSSND+uIpLm0v/u1JEnSEGxCJElSL/bahCSZSvKeUSUjafJYRyTNZ69NSFXtAg5vToksSQtmHZE0n2GuonsF8Pkk5zI4IRAAVfXmeR8hSXd1BdYRSTMM04Rc2/wsY/9OLiTpwGUdkTTLMIfovhYgybqZ12WQpGFYRyTNZZ9HxyR5eJJvApc19x+U5H91npmkiWEdkTSXYQ7RfQvws8BNAFX1NQaX0JakYb0F64ikGYY6T0hVXTVj0q4OcpE0wawjkmYaZsfUq5L8DFDNIXYvphlSlaQhWUckzTLMSMhvAC9icMGpqxlcKOpFHeYkafJYRyTNMszRMd8Hnj2CXCRNKOuIpLnM24QkeRtQ882vqhd3kpGkiWEdkbQ3exsJuXBkWUiaVNYRSfOatwmpqrNGmYikyWMdkbQ3+9wnJMnhwO8DJwCr90yvqsd1mNddcyiY2j7viO7i43Z0gGB2tx9z2ZgdzLit0kncbBvqqPIl4Y7d3VyvbUcHR7bW/N+YtGIp1JGpbbDhivY3zlvWdPNe37m2/ddk9U3d5FodbZYHXdNN3O+f1EGRBrbvmmo95u27VrUeE+DOav9vULXw9+wwb52zGRxKd2/gtQwuRPUvC16TpAOZdUTSLMM0IYdV1RnAjqo6v6qeDzys47wkTRbriKRZhjlZ2Y7m9+YkP8fgSphHdZeSpAlkHZE0yzBNyOuSHAz8DvA2YAPwsk6zkjRprCOSZhnmZGXnNTd/ADy223QkTSLriKS57HOfkCTHJflkkkua+w9M8uruU5M0KawjkuYyzI6p/xt4Jc13ulX1deBZXSYlaeJYRyTNMkwTsraqvjxj2s4ukpE0sawjkmYZpgn5fpJjaa7/kOSXgM2dZiVp0lhHJM0yzNExLwJOB+6X5Brg34FTO81K0qSxjkiaZZijYy4HHp9kHbCsqrZ0n5akSWIdkTSXeZuQJL89z3QAqurNHeUkaUJYRyTtzd5GQtY3v+8LPAQ4t7n/NOCzXSYlaWJYRyTNa94mpKpeC5Dk48BJe4ZPk/wR8P6RZCdprFlHJO3NMEfH/Diwfdr97cAxnWQjaVJZRyTNMszRMe8GvpzkQwwOr/sF4KxOs5I0aawjkmYZ5uiY/5nkH4BHNpOeV1Vf7TYtSZPEOiJpLsOMhFBVFwEXtbniJFPAhcA1VfXUNmNLWnqsI5JmGmafkK68BLisx/VLGn/WEWmM9dKEJDkK+Dng//Sxfknjzzoijb++RkLeArwc2N3T+iWNv7dgHZHG2lD7hLQpyVOBG6rqK0kes5flTgNOA1i5diPZVa3nsmL7vpdZlPZTpabajwmwbGcHyQJTd6aTuF3Zsav9P/CK7Go9JsCOav9/bjfvgu4spo6sWnUw66/e1noutx+1pvWYAMs6uMbwjrXtxwRYsbWbuLtWdhN32fZu6tMd21e0HvPg5T9sPSZAF9VpMXWkj5GQk4GnJ7kCeB/wuCTvmblQVZ1eVZuqatOKVQeNOkdJS9uC68jKFetGnaOkfRh5E1JVr6yqo6rqGOBZwKeqyqtpShqadUSaDH0eHSNJkg5gI98nZLqq+gzwmT5zkDTerCPS+HIkRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9WJ53wkMI7uL5XdW63GXbd/dekyAmkrrMdNNqlRHbeiyHd3EJe2/DwC2blvZesxlHb1oK9L+i9b+O3bp2X7wMq584urW407d0XpIAHataj/mumu72X52runmHTS1rZOwZGc3+Va1H/eG7etbjwmwMh38n1pETEdCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL2xCJElSL0behCQ5Osmnk1yW5NIkLxl1DpLGm3VEmgzLe1jnTuB3quqiJOuBryT5RFV9s4dcJI0n64g0AUY+ElJVm6vqoub2FuAy4MhR5yFpfFlHpMnQx0jIjyQ5Bngw8KU55p0GnAaweuogDvqX73WRQPsxgdp6R+sxd92+tfWYAOze1UnY1atWdRL3kIcc30ncmx5wSOsx33bfJ7UeE+D2U1a3HvOGnRe0HnNUhq4jyzdw7Fk3tJ/Aim7K6O7VHcTtqOZR1U3Yr1zaSdyNJ5/YSdybv3Nw6zE/fMJDW48JsOaUHa3HvGHnpxf8mN52TE1yEPBB4KVVddvM+VV1elVtqqpNK5etGX2Ckpa8BdWRqbWjT1DSXvXShCRZwaBwnF1V5/SRg6TxZh2Rxl8fR8cEOAO4rKrePOr1Sxp/1hFpMvQxEnIy8F+BxyW5uPl5Sg95SBpf1hFpAox8x9Sq+hzQ0d5Rkg4E1hFpMnjGVEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1AubEEmS1ItUVd857FOSLcC3+s5jSHcDvt93EgswTvmOU64wXvnet6rW951El6wjnRqnfMcpVxivfBdcR5Z3lUnLvlVVm/pOYhhJLhyXXGG88h2nXGG88k1yYd85jIB1pCPjlO845Qrjle9i6ohfx0iSpF7YhEiSpF6MSxNyet8JLMA45Qrjle845Qrjle845bpY4/QcxylXGK98xylXGK98F5zrWOyYKkmSJs+4jIRIkqQJYxMiSZJ6seSakCR/k+SGJJdMm/amJP+a5OtJPpRkY48p/shcuU6b97tJKsnd+shtLvPlm+S3knwryaVJ3thXftPN8z44MckXk1yc5MIkD+0zxz2SHJ3k00kua/6GL2mmH5rkE0m+0/w+pO9cYa/5LsntbDGsI92xjnTjgK0jVbWkfoBHAScBl0yb9kRgeXP7DcAb+s5zvlyb6UcD/wR8D7hb33nu42/7WOCfgVXN/bv3nedecv048OTm9lOAz/SdZ5PLEcBJze31wLeBE4A3Aq9opr9iCb1v58t3SW5nLb5/luTzs46MPFfrSLf5Lmg7W3IjIVX1WeDmGdM+XlU7m7tfBI4aeWJzmCvXxl8ALweW1F6/8+T7m8CfVtW2ZpkbRp7YHObJtYANze2DgWtHmtQ8qmpzVV3U3N4CXAYcCTwDOKtZ7Czg53tJcIb58l2q29liWEe6Yx3pxoFaR5ZcEzKE5wP/0HcS80nydOCaqvpa37kM6TjgkUm+lOT8JA/pO6G9eCnwpiRXAX8GvLLfdGZLcgzwYOBLwD2qajMMNljg7j2mNqcZ+U63pLezFizp52cd6dRLsY60an/qyFg1IUn+ANgJnN13LnNJshb4A+A1feeyAMuBQ4CHAb8H/F2S9JvSvH4TeFlVHQ28DDij53zuIslBwAeBl1bVbX3nsy/z5bvUt7P9tdSfn3Wkc9aRFu1vHRmbJiTJc4CnAs+u5sumJehY4N7A15JcwWAY6qIkP9ZrVnt3NXBODXwZ2M3ggklL0XOAc5rb7weWxA5lAElWMNgQz66qPTlen+SIZv4RwJIYooZ58x2X7WzRxuT5WUe6ZR1pSRt1ZCyakCRPAn4feHpV3dF3PvOpqm9U1d2r6piqOobBhnlSVV3Xc2p78/fA4wCSHAesZOlesfFa4NHN7ccB3+kxlx9pPvGdAVxWVW+eNutcBgWP5veHR53bXObLd1y2s8Ual+dnHemcdaQFrdWRvvewnfkDvBfYDOxgsPG9APgucBVwcfPzjr7znC/XGfOvYGnt1T7X33Yl8B7gEuAi4HF957mXXB8BfAX4GoPvHn+q7zybXB/BYGe3r097jz4FOAz4JIMi90ng0L5z3Ue+S3I7a/H9sySfn3Vk5LlaR7rNd0HbmadtlyRJvRiLr2MkSdLksQmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAkRST6TZFPfeUgaX9YRLYZNiDqXZKrvHCSNN+vIZLIJOUAkOSbJvyY5K8nXk3yguUbFzOXenuTCJJcmeW0z7ZQkH5q2zBOSnNPcfmKSLyS5KMn7m+sIkOSKJK9J8jngmTPW8cwklyT5WpLPNtOem+Svpi1zXpLHNLdvT/KGJF9J8s9JHtp86rq8udCXpBGwjqhtNiEHlvsCp1fVA4HbgP8+xzJ/UFWbgAcCj07yQOBTwPFJDm+WeR7wziR3A14NPL6qTgIuBH57Wqw7q+oRVfW+Get4DfCzVfUgYJiNfx3wmar6KWAL8DrgCcAvAH88xOMltcc6otbYhBxYrqqqzze338PgtLsz/XKSi4CvAvcHTqjBaXXfDZyaZCPwcAaXZ34YcALw+SQXM7iuwb2mxfrbefL4PHBmkhcCwwyxbgf+sbn9DeD8qtrR3D5miMdLao91RK1Z3ncCGqmZ5+i/y/0k9wZ+F3hIVd2S5ExgdTP7ncBHgDuB91fVzuYCRp+oql+dZ31b50yi6jeS/DTwc8DFSU5kcMnn6U3x6mm3d9R/XF9gN7CtibM7ie9habSsI2qNIyEHlh9P8vDm9q8Cn5sxfwODDf4HSe4BPHnPjKq6lsHVJ18NnNlM/iJwcpL7ACRZ21xBc6+SHFtVX6qq1zC40ubRDC7SdWKSZUmOZgldXlvSXVhH1Bq7vwPLZcBzkvw1gysyvn36zKr6WpKvApcClzMY7pzubODwqvpms/yNSZ4LvDfJqmaZVwPf3kceb0ryk0AYXBXya830f2cwNLrnSpySlh7riFrjVXQPEEmOAc6rqgfsR4y/Ar5aVWe0lpiksWEdUdscCdFQknyFwRDr7/Sdi6TxZB3RTI6ESJKkXrhjqiRJ6oVNiCRJ6oVNiCRJ6oVNiCRJ6oVNiCRJ6sX/B7YeUa6H6CAjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(policy.argmax(-1))\n",
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23:18:33 [INFO] average episode reward = -0.12 ± 0.96\n"
     ]
    }
   ],
   "source": [
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_policy(env, policy)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Off-Policy Monte Carlo Update\n",
    "\n",
    "Monte Carlo evaluation with importance sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_monte_carlo_importance_sample(env, policy, behavior_policy,\n",
    "        episode_num=500000):\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # play episode using behavior policy\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n,\n",
    "                    p=behavior_policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # finish the episode\n",
    "        g = reward # return\n",
    "        rho = 1. # importance sampling ratio\n",
    "        for state, action in reversed(state_actions):\n",
    "            c[state][action] += rho\n",
    "            q[state][action] += (rho / c[state][action] * (g - q[state][action]))\n",
    "            rho *= (policy[state][action] / behavior_policy[state][action])\n",
    "            if rho == 0:\n",
    "                break # early stop\n",
    "    return q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy = np.zeros((22, 11, 2, 2))\n",
    "policy[20:, :, :, 0] = 1 # stand when >=20\n",
    "policy[:20, :, :, 1] = 1 # hit when <20\n",
    "behavior_policy = np.ones_like(policy) * 0.5\n",
    "q = evaluate_monte_carlo_importance_sample(env, policy, behavior_policy)\n",
    "v = (q * policy).sum(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAet0lEQVR4nO3debhkd13n8fen+/btNUlDEiIkgbCF9cEQGwTDGiACsqjjgkNm2CTq8MgiiqCIg8OIgCKoz4CZiSRABhQIGEBllRAYCIYQQiCyGEM6C9lDlk4vt+93/qjTeHOX7rq3z7nnVvX79Tz1dNU5p77nW9V1vvdbvzpLqgpJkqTltqrvBCRJ0oHJJkSSJPXCJkSSJPXCJkSSJPXCJkSSJPXCJkSSJPXCJuQAlOS2JPfZy/zLkjx5OXOSNJ6sN9obm5ADUFVtqqpLAZKcnuQNy7HeJM9P8oXlWJeklaGveqPRYBMiSZJ6YRMyJpK8IMlHZzz+XpK/m/F4a5LjmvuV5H5JTgGeC7yqGTL96IyQxyW5KMkPk/xtknUzYr24iX9jkrOT3KOZfkwTe2LGsp9L8qtJHgS8E3h0s66b9/I6Lklya5JLk/zarPnPTnJhkluS/FuSpzbTD0lyWpKrk1yZ5A1JVi/5DZW0oOWsN7PWe98kn01yQ5Lrk5yZZPOM+UcnOSvJdc0yfzVj3gub2nJTkk8kuVdrb4iWzCZkfJwDPDbJqiR3B9YAJwA0v8duAi6a+YSqOhU4E3hzM2T6zBmzfwl4KnBv4GHA85tYJwJvbObfHfg+8P59JVdVlwC/DnypWdfmBRa9FngGcDDwAuDPkxzfrPuRwLuB3wE2A48DLmuedwYwBdwPeDhwEvCr+8pL0pIsS72ZRxjUn3sADwKOBv57s97VwMcY1KRjgCNpalOSnwV+D/h54HDgXOB9S3vpatPEvhfRKKiqS5PcChwHHAt8gsG3iwcCjwbOrarpRYT8i6q6CqD5xnJcM/25wN9U1QXNvNcANyU5pqXX8fEZD89J8kngscAFwIuadX+qmX9lk8MRwNOAzVV1B3B7kj8HTgH+uo28JP2HZaw3s9f7PeB7zcPrkrwV+MPm8SMZNCe/U1VTzbQ9+6D9GvDG5ssQSf4Y+L0k96qq7y8iT7XMJmS8nAM8gcFowDnAzcDjGRSFcxYZ6wcz7m9jsHHT/HvBnhlVdVuSGxh867hyKUnPlORpDIrKsQxG6jYA32hmHw38wzxPuxeDb2JXJ9kzbRWwdX/zkbSg5ag3d5LkbsBfMPhichCD7fymZvbRwPdnNCAz3Qt4e5I/mxmOQd2yCemRP8eMlz1F4bHN/XMYFIXHs3BRWOxllK9isEEDkGQjcCiDBuT2ZvKGGcv/2LDrSrIW+BDwp8ARzU82/8CgWMCgqbjvPE/dCuwADquqzc3t4Kp6yLAvStKiLUe9me2NTYyHVdXBwMncuT7cc+Y+aTNsBX5tRn3YXFXrq+r/7Wc+2k82IePlHOCJwPqquoLB755PZdAkfG2B51wDLHgM/zz+L/CCJMc1TcMfA+dV1WVVdR2DZuTkJKuTvJA7Nw3XAEclmVwg9iSwFrgOmGpGRU6aMf+0Zt1Pan6LPjLJA6vqauCTwJ8lObiZd98kj1/E65K0OMtRb2Y7CLgNuDnJkQz2D9vjK8DVwJ8k2ZhkXZITmnnvBF6T5CHwox3Zf3E/8lBLbELGSFV9h8EGem7z+BbgUuCLVbV7gaedBjw4yc1JPjLEOj4D/AGDEYurGTQZz5mxyIsZFIYbgIcAM79pfBb4JvCDJNfPE/tW4KXA3zEYYv3PwNkz5n+FZmdV4IcMiuCeUZn/yqCJ+Vbz3A8y2HFWUgeWo97M4/XA8Qy2/48DZ83IZzfwTAY/D10OXAH8cjPvw8CbgPcnuQW4mMF+ZOpZqvZ3dEySJGnxHAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9GImTlU1ObKj1k5v7TmN404s5UWDPpkdsx+RFnYRxEbp4H+7XzaVr7r3upn0vtEhbt05xw43T2feSo2ti3caaPOiurcdd1dFHctWODgJ39T+cbgJ3VZ1W7Vzo4J39UxPtf6+vI7vJ9T7rbmw95tatU9y4yDoyEk3I+snNPOqBL24/8FQ31WPVtu2dxO3Ezl3dxO3oqKva3s17W3e0Hzenbm49JsC77/eB1mOe9PQ5R0yPncmD7soDf+4V7ce9rZvP+kGX3r7vhRZpem03Jb8mOmpCOmpu1l5xcydxpw7d1HrM3f+j/S8dAO97QPuXzvnpJdQRf46RJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm9sAmRJEm96KwJSfI3Sa5NcvGMaXdN8qkk323+vUtX65c0+qwj0njrciTkdOCps6a9GvhMVd0f+EzzWJIWcjrWEWlsddaEVNXngRtnTX42cEZz/wzgZ7tav6TRZx2RxtvEMq/viKq6GqCqrk5yt4UWTHIKcArAujWHQFXrydSa1a3HBKh1k+0HndrdfkygNq3rJO6qG27pJG42bewkLhPtbwqvvOcnWo8JcNjq9t+DiTl/51e0JdWRtWs3s/l7O1pPZmpDN3XkyhMPaj3m5u91U0c2Xr6tk7jZPd1J3BsfeXgncddf3/77+4b7fKT1mACHrlrfeswJsujnrNgdU6vq1KraUlVbJic29J2OpBF0pzoy2VEDK2nJlrsJuSbJ3QGaf69d5vVLGn3WEWlMLHcTcjbwvOb+84C/X+b1Sxp91hFpTHR5iO77gC8BD0hyRZIXAX8CPCXJd4GnNI8laV7WEWm8dbZjalX9ygKzntTVOiWNF+uINN5W7I6pkiRpvNmESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXtiESJKkXkz0ncAwanXYvWlt32kMLVXtx2w9YhN3x1QncWv79k7idibtv8OHrtrWekyA3bWm9ZhF+5/ZlWbXhnDt8etaj7tqV+shATjq07e0HnPrUw5uPSbAxO3tv68Aq3Z387k85HvdbJvVQR3ZVd38mV6d6dZjZgl/qRwJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJvbAJkSRJveilCUnyiiTfTHJxkvclWddHHpJGl3VEGn3L3oQkORJ4KbClqh4KrAaes9x5SBpd1hFpPPT1c8wEsD7JBLABuKqnPCSNLuuINOImlnuFVXVlkj8FLgfuAD5ZVZ/c63NWhamN7ae69prbW48JwO5qPWTu2NF6TAC2dxO3Jrr5aKWjuDU11XrM22tN6zEBptjdesz2P7HdWkodWTUF669r/5UetLWjbTNpPeTB359uPSbA+qtu6yTu1MHd/MI2taGbbXPdd37Qesxbprv6lXFbR3EXp4+fY+4CPBu4N3APYGOSk+dZ7pQk5yc5f9fOjpoFSSNpKXVkart1RFpp+vg55snAv1fVdVW1CzgL+KnZC1XVqVW1paq2rJncuOxJSlrRFl1HJtZZR6SVpo8m5HLgUUk2JAnwJOCSHvKQNLqsI9IYWPYmpKrOAz4IXAB8o8nh1OXOQ9Loso5I42HZd0wFqKo/BP6wj3VLGg/WEWn0ecZUSZLUC5sQSZLUi33+HJPk5+eZ/EPgG1V1bfspSRo31hFJ8xlmn5AXAY8G/rl5/ATgy8CxSf6oqt7TUW6Sxod1RNIcwzQh08CDquoagCRHAO8AfhL4PGDxkLQv1hFJcwyzT8gxewpH41rg2Kq6EdjVTVqSxox1RNIcw4yEnJvkY8AHmsf/Cfh8ko3AzV0lJmmsWEckzTFME/ISBgXjBCDAu4EPVVUBT+wwN0njwzoiaY59NiFNkfhgc5OkRbOOSJrPPvcJSfLzSb6b5IdJbklya5JbliM5SePBOiJpPsP8HPNm4JlV5cWhJC2VdUTSHMMcHXONhUPSfrKOSJpjmJGQ85P8LfARYMeeiVV1VldJSRo71hFJcwzThBwMbANOmjGtAIuHpGFZRyTNMczRMS9YjkQkjS/riKT5LNiEJHlVVb05yV8y+MZyJ1X10k4zkzTyrCOS9mZvIyF7diI7fzkS2ZtMTTN53R2tx62JYfbLXbxkTq3db7VhXesxATI93U3cVd28t53Z2f6Zw7fuOrT1mAAPnbyq9Zg1tz9oy4qpIxWYWt9+3G1HTLYfFFi9c03rMVdNtR4SgDvusamTuNNr00ncjZfd1knc3Ydvbj3m9mr/cwBw2/T21mPuZvF/TxZsQqrqo83dc6vq0qUmJenAZR2RtDfD7Jh6epIjgX9hcLXLc6vqG92mJWnMWEckzTHMjqmPSzIJPAJ4AvDxJJuq6q5dJydpPFhHJM1nn01IkscAj21um4GPAed2m5akcWIdkTSfYX6OOYfBTmVvBP6hqnZ2m5KkMWQdkTTHME3IoQwuv/044KVJpoEvVdUfdJqZpHFiHZE0xzD7hNyc5FLgaOAo4KeAbo4ZkjSWrCOS5jPMPiH/Bnwb+ALwTuAFDqVKWgzriKT5DPNzzP2rqpszWkk6UFhHJM0xzGkt75Hkw0muTXJNkg8lOarzzCSNE+uIpDmGaULeBZwN3AM4EvhoM02ShmUdkTTHME3I4VX1rqqaam6nA4d3nJek8WIdkTTHME3I9UlOTrK6uZ0M3NB1YpLGinVE0hzDNCEvBH4J+AFwNfALzTRJGpZ1RNIcw5wn5HLgWcuQi6QxZR2RNJ9hzhNyOPBi4JiZy1eV32IkDcU6Imk+w5wn5O8ZXGjq08DubtORNKasI5LmGKYJ2VBVv9t5JpLGmXVE0hzD7Jj6sSRPb3OlSTYn+WCSf01ySZJHtxlf0opjHZE0x4IjIUluBQoI8HtJdgC7msdVVQfvx3rfDvxTVf1Ckklgw37EkrRCWUck7c2CTUhVHdTFCpMczOBy3s9v1rMT8EJW0hiyjkjam2GOjjkBuLCqbm9OMHQ88LbmkLuluA9wHfCuJD8OfBV4WVXdPmu9pwCnAKybPIRau3qJq1vY6pu2tR4TYHrD2tZjrtq2vfWYAKxu/30FYNdUJ2Hrttv3vdBSrErrIW/e3c0X823T7e/XOV3VesyZVkIdmdx4F9Zsa/91bri2m95n8ltXtB4zE8PsBrh4dzzo7p3EXfuPX+0kLsc9uJOwmW7/Go237l7fekyA7dV+HVnK1jXMPiHvALY1G/qrgO8D71nCuvaYYFCA3lFVDwduB149e6GqOrWqtlTVljVrNu7H6iStAL3XkYm11hFppRmmCZmqqgKeDby9qt4O7M8Q6xXAFVV1XvP4gwyKiaTxZR2RNMcwTcitSV4DnAx8PMlqYM1SV1hVPwC2JnlAM+lJwLeWGk/SSLCOSJpjmCbkl4EdwIuaDf9I4C37ud7fBM5MchFwHPDH+xlP0spmHZE0xzDXjvkB8NYZjy8H3r0/K62qC4Et+xND0uiwjkiazzAjIZIkSa2zCZEkSb3YaxOSZHWS9y5XMpLGj3VE0kL22oRU1W7g8OaUyJK0aNYRSQsZ5vR5lwFfTHI2gxMCAVBVb13wGZJ0Z5dhHZE0yzBNyFXNbRX7d3IhSQcu64ikOYY5RPf1AEk2zr4ugyQNwzoiaT77PDomyaOTfAu4pHn840n+V+eZSRob1hFJ8xnmEN23AT8N3ABQVV9ncAltSRrW27COSJplqPOEVNXWWZPavwawpLFmHZE02zA7pm5N8lNANYfYvZRmSFWShmQdkTTHMCMhvw68hMEFp65gcKGol3SYk6TxYx2RNMcwR8dcDzx3GXKRNKasI5Lms2ATkuQvgVpoflW9tJOMJI0N64ikvdnbSMj5y5aFpHFlHZG0oAWbkKo6YzkTkTR+rCOS9maf+4QkORz4XeDBwLo906vqxA7zurMqsqODo/lWpf2YwOqbb2s/aLrJlZ27uolbC47A75ds2thJ3Okbbmw95q4a5uCzxVuXoY6sX5R09fn6j/i915HVu4pNV+xsPe7klTe3HhNg54OPaj3mtiO6uYbg+mu7qSN1wnGdxF111U2dxGWq/b9T39n+Y63HBJje9G+tx1xK1R+mmp3J4FC6ewOvZ3Ahqn9ZwrokHbisI5LmGKYJObSqTgN2VdU5VfVC4FEd5yVpvFhHJM0xzHjxnnG2q5P8DIMrYbY/TihpnFlHJM0xTBPyhiSHAK8E/hI4GHhFp1lJGjfWEUlzDHOyso81d38IPLHbdCSNI+uIpPnsc5+QJMcm+UySi5vHD0vy2u5TkzQurCOS5jPMjqn/G3gNzW+6VXUR8Jwuk5I0dqwjkuYYpgnZUFVfmTVtqotkJI0t64ikOYZpQq5Pcl+a85Ak+QXg6k6zkjRurCOS5hjm6JiXAKcCD0xyJfDvwMmdZiVp3FhHJM0xzNExlwJPTrIRWFVVt3aflqRxYh2RNJ8Fm5Akv7XAdACq6q0d5SRpTFhHJO3N3kZCDmr+fQDwCODs5vEzgc93mZSksWEdkbSgBZuQqno9QJJPAsfvGT5N8t+BDyxLdpJGmnVE0t4Mc3TMPYGZ17/eCRzTSTaSxpV1RNIcwxwd8x7gK0k+zODwup8Dzug0K0njxjoiaY5hjo75n0n+EXhsM+kFVfW1btOSNE6sI5LmM8xICFV1AXBBmytOsho4H7iyqp7RZmxJK491RNJsw+wT0pWXAZf0uH5Jo886Io2wXpqQJEcBPwP8nz7WL2n0WUek0dfXSMjbgFcB0z2tX9LoexvWEWmkDbVPSJuSPAO4tqq+muQJe1nuFOAUgHWTh8BE+/1SrV7dekyA6btuaj3m6hs6Ost1VTdh16/tJG6279z3QkuJu35dJ3G7cHu1/zd3uqPPQVeWUkfWrj2E7G7/dd72kMNajwmw7podrcc85OyLWo8JMHX8sZ3Enbium7q3u4MaDbB7w2TrMVdzeesxAXZ3sc0vIWYfIyEnAM9KchnwfuDEJO+dvVBVnVpVW6pqy5qJjcudo6SVbdF1ZHKNdURaaZa9Camq11TVUVV1DPAc4LNV5dU0JQ3NOiKNhz6PjpEkSQewZd8nZKaq+hzwuT5zkDTarCPS6HIkRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9cImRJIk9WKi7wSGUzA13X7YjlqwVbftaD1mbVjXekyA3HRLN3HvaP89AKipqU7ismp16yF3k9ZjAqxL+3FXdRBzpanVYefmNa3HnV7d0XvXQdjbT3po+0GBiW0d1GdgcsfOTuKu3rmrk7hTBx/WesxDJu5oPSbA6i62+SXEdCREkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1wiZEkiT1YtmbkCRHJ/nnJJck+WaSly13DpJGm3VEGg8TPaxzCnhlVV2Q5CDgq0k+VVXf6iEXSaPJOiKNgWUfCamqq6vqgub+rcAlwJHLnYek0WUdkcZDHyMhP5LkGODhwHnzzDsFOAVg3cTBrNq2o/3179jZekyAmljdftBrrms/JrD7ju2dxM1ENx+trFvbSdzdN9zYesy3f+EprccEWPPY3a3HvHH3+a3HXC7D1pHJ9ZuZnkjr619/fTd1ZOddJluPedB5l7ceE4COtvft9z+ik7iTN2zrJO7aS65sPeZff+7E1mMC7Hp8+3+nrp86d9HP6W3H1CSbgA8BL6+qW2bPr6pTq2pLVW2ZnNiw/AlKWvEWU0fWrN20/AlK2qtempAkaxgUjjOr6qw+cpA02qwj0ujr4+iYAKcBl1TVW5d7/ZJGn3VEGg99jIScAPwX4MQkFza3p/eQh6TRZR2RxsCy75haVV8A2t87TNIBwzoijQfPmCpJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknphEyJJknqRquo7h31Kcivw7b7zGNJhwPV9J7EIo5TvKOUKo5XvA6rqoL6T6JJ1pFOjlO8o5Qqjle+i68hEV5m07NtVtaXvJIaR5PxRyRVGK99RyhVGK98k5/edwzKwjnRklPIdpVxhtPJdSh3x5xhJktQLmxBJktSLUWlCTu07gUUYpVxhtPIdpVxhtPIdpVyXapRe4yjlCqOV7yjlCqOV76JzHYkdUyVJ0vgZlZEQSZI0ZmxCJElSL1ZcE5Lkb5Jcm+TiGdPekuRfk1yU5MNJNveY4o/Ml+uMeb+dpJIc1kdu81ko3yS/meTbSb6Z5M195TfTAp+D45J8OcmFSc5P8sg+c9wjydFJ/jnJJc17+LJm+l2TfCrJd5t/79J3rrDXfFfkdrYU1pHuWEe6ccDWkapaUTfgccDxwMUzpp0ETDT33wS8qe88F8q1mX408Ang+8Bhfee5j/f2icCngbXN47v1nedecv0k8LTm/tOBz/WdZ5PL3YHjm/sHAd8BHgy8GXh1M/3VK+hzu1C+K3I7a/HzsyJfn3Vk2XO1jnSb76K2sxU3ElJVnwdunDXtk1U11Tz8MnDUsic2j/lybfw58CpgRe31u0C+vwH8SVXtaJa5dtkTm8cCuRZwcHP/EOCqZU1qAVV1dVVd0Ny/FbgEOBJ4NnBGs9gZwM/2kuAsC+W7UrezpbCOdMc60o0DtY6suCZkCC8E/rHvJBaS5FnAlVX19b5zGdKxwGOTnJfknCSP6DuhvXg58JYkW4E/BV7TbzpzJTkGeDhwHnBEVV0Ngw0WuFuPqc1rVr4zrejtrAUr+vVZRzr1cqwjrdqfOjJSTUiS3wemgDP7zmU+STYAvw+8ru9cFmECuAvwKOB3gL9Lkn5TWtBvAK+oqqOBVwCn9ZzPnSTZBHwIeHlV3dJ3PvuyUL4rfTvbXyv99VlHOmcdadH+1pGRaUKSPA94BvDcan5sWoHuC9wb+HqSyxgMQ12Q5Md6zWrvrgDOqoGvANMMLpi0Ej0POKu5/wFgRexQBpBkDYMN8cyq2pPjNUnu3sy/O7AihqhhwXxHZTtbshF5fdaRbllHWtJGHRmJJiTJU4HfBZ5VVdv6zmchVfWNqrpbVR1TVccw2DCPr6of9Jza3nwEOBEgybHAJCv3io1XAY9v7p8IfLfHXH6k+cZ3GnBJVb11xqyzGRQ8mn//frlzm89C+Y7KdrZUo/L6rCOds460oLU60vcetrNvwPuAq4FdDDa+FwHfA7YCFza3d/ad50K5zpp/GStrr/b53ttJ4L3AxcAFwIl957mXXB8DfBX4OoPfHn+i7zybXB/DYGe3i2Z8Rp8OHAp8hkGR+wxw175z3Ue+K3I7a/HzsyJfn3Vk2XO1jnSb76K2M0/bLkmSejESP8dIkqTxYxMiSZJ6YRMiSZJ6YRMiSZJ6YRMiSZJ6YRMiknwuyZa+85A0uqwjWgqbEHUuyeq+c5A02qwj48km5ACR5Jgk/5rkjCQXJflgc42K2cu9I8n5Sb6Z5PXNtCcl+fCMZZ6S5Kzm/klJvpTkgiQfaK4jQJLLkrwuyReAX5y1jl9McnGSryf5fDPt+Un+asYyH0vyhOb+bUnelOSrST6d5JHNt65Lmwt9SVoG1hG1zSbkwPIA4NSqehhwC/Df5lnm96tqC/Aw4PFJHgZ8FnhQksObZV4AvCvJYcBrgSdX1fHA+cBvzYi1vaoeU1Xvn7WO1wE/XVU/Dgyz8W8EPldVPwHcCrwBeArwc8AfDfF8Se2xjqg1NiEHlq1V9cXm/nsZnHZ3tl9KcgHwNeAhwINrcFrd9wAnJ9kMPJrB5ZkfBTwY+GKSCxlc1+BeM2L97QJ5fBE4PcmLgWGGWHcC/9Tc/wZwTlXtau4fM8TzJbXHOqLWTPSdgJbV7HP03+lxknsDvw08oqpuSnI6sK6Z/S7go8B24ANVNdVcwOhTVfUrC6zv9nmTqPr1JD8J/AxwYZLjGFzyeWZTvG7G/V31H9cXmAZ2NHGmk/gZlpaXdUStcSTkwHLPJI9u7v8K8IVZ8w9msMH/MMkRwNP2zKiqqxhcffK1wOnN5C8DJyS5H0CSDc0VNPcqyX2r6ryqeh2DK20ezeAiXcclWZXkaFbQ5bUl3Yl1RK2x+zuwXAI8L8lfM7gi4ztmzqyqryf5GvBN4FIGw50znQkcXlXfapa/LsnzgfclWdss81rgO/vI4y1J7g+EwVUhv95M/3cGQ6N7rsQpaeWxjqg1XkX3AJHkGOBjVfXQ/YjxV8DXquq01hKTNDKsI2qbIyEaSpKvMhhifWXfuUgaTdYRzeZIiCRJ6oU7pkqSpF7YhEiSpF7YhEiSpF7YhEiSpF7YhEiSpF78fyvqdeA6vE+pAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Monte Carlo update with importance sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monte_carlo_importance_sample(env, episode_num=500000):\n",
    "    policy = np.zeros((22, 11, 2, 2))\n",
    "    policy[:, :, :, 0] = 1.\n",
    "    behavior_policy = np.ones_like(policy) * 0.5 # soft policy\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # play using behavior policy\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n,\n",
    "                    p=behavior_policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # finish the episode\n",
    "        g = reward # return\n",
    "        rho = 1. # importance sampling ratio\n",
    "        for state, action in reversed(state_actions):\n",
    "            c[state][action] += rho\n",
    "            q[state][action] += (rho / c[state][action] * (g - q[state][action]))\n",
    "            # improve the policy\n",
    "            a = q[state].argmax()\n",
    "            policy[state] = 0.\n",
    "            policy[state][a] = 1.\n",
    "            if a != action: # early stop\n",
    "                break\n",
    "            rho /= behavior_policy[state][action]\n",
    "    return policy, q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy, q = monte_carlo_importance_sample(env)\n",
    "v = q.max(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ/0lEQVR4nO3debSkdX3n8feHbhAbQRRaQjeNbVDccrAlrRERF1xw1zhqdGRG0YhmPEGNcUGNxhzjHtToGQ0zKKiMG6IimoCiLDoKaVpW2y2IsjSrsogj63f+qKfN5W5dt++t+t2qfr/OuedWPc9z6/ncPl3f87lPPVVPqgpJkqRh26Z1AEmStHWyhEiSpCYsIZIkqQlLiCRJasISIkmSmrCESJKkJiwhW6Ekv03yx7OsvzjJE4aZSdJ4ct5oNpaQrVBV3a2qLgJIcnSSdw5jv0lekuS7w9iXpMWh1bzRaLCESJKkJiwhYyLJIUm+NuH+z5N8YcL9S5Ks6W5XkvsmORR4EfCG7pDp1yY85Jok5yW5Psnnk2w/4bFe3j3+r5OckGRFt3x199hLJ2x7apK/TPJA4OPAft2+rpvl99iQ5MYkFyV5xaT1z0pyTpIbkvxHkid3y++e5KgkG5NcluSdSZZs8T+opBkNc95M2u9eSb6d5Nok1yQ5NsnOE9avSnJ8kqu7bT46Yd1Lu9nymyQnJbn3gv2DaItZQsbHacABSbZJsjuwLbA/QPd67N2A8yb+QFUdCRwLvK87ZPqMCaufDzwZuA+wD/CS7rEOBN7drd8d+CXwuc2Fq6oNwCuB73f72nmGTa8Cng7sBBwCfDDJvt2+Hw58Cng9sDPwaODi7ueOAW4D7gs8FHgS8JebyyVpiwxl3kwj9ObPCuCBwCrg77v9LgFOpDeTVgMr6WZTkmcDbwaeAywHzgA+u2W/uhbS0s1volFQVRcluRFYA+wNnETvr4sHAPsBZ1TVHXN4yH+uqssBur9Y1nTLXwR8oqrWd+sOB36TZPUC/R5fn3D3tCQnAwcA64GXdfv+Zrf+si7DbsBTgJ2r6v8BNyX5IHAo8C8LkUvSfxrivJm8358DP+/uXp3kCODt3f2H0ysnr6+q27plm85BewXw7u6PIZK8C3hzkntX1S/nkFMLzBIyXk4DHkvvaMBpwHXAY+gNhdPm+FhXTLj9O3pPbrrv6zetqKrfJrmW3l8dl21J6ImSPIXeUNmb3pG6ZcD53epVwDem+bF70/tLbGOSTcu2AS6Zbx5JMxrGvLmTJPcC/pneHyY70nue/6ZbvQr45YQCMtG9gQ8n+aeJD0dvbllCGvLlmPGyaSgc0N0+jd5QeAwzD4W5Xkb5cnpPaACS7ADsQq+A3NQtXjZh+z/qd19J7gJ8CfgAsFv3ks036A0L6JWKvab50UuAm4Fdq2rn7munqnpwv7+UpDkbxryZ7N3dY+xTVTsBB3Pn+bDnxHPSJrgEeMWE+bBzVd21qv7vPPNoniwh4+U04HHAXavqUnqvez6ZXkn44Qw/cyUw43v4p/F/gEOSrOlKw7uAM6vq4qq6ml4ZOTjJkiQv5c6l4UpgjyTbzfDY2wF3Aa4GbuuOijxpwvqjun0/vnstemWSB1TVRuBk4J+S7NSt2yvJY+bwe0mam2HMm8l2BH4LXJdkJb3zwzY5C9gIvCfJDkm2T7J/t+7jwOFJHgx/OJH9efPIoQViCRkjVfVTek/QM7r7NwAXAd+rqttn+LGjgAcluS7JV/rYxynA39E7YrGRXsl4wYRNXk5vMFwLPBiY+JfGt4ELgSuSXDPNY98IHAZ8gd4h1v8KnDBh/Vl0J6sC19MbgpuOyvx3eiXmR93PHkfvxFlJAzCMeTONdwD70nv+fx04fkKe24Fn0Ht56FfApcBfdOu+DLwX+FySG4AL6J1HpsZSNd+jY5IkSXPnkRBJktSEJUSSJDVhCZEkSU1YQiRJUhMj8WFlu95zSa1etW3rGJqDn563bPMbadH4PTdxS92czW85upwjGrStfe5tyRwZiRKyetW2nHXSqtYxNAcHrVjTOoLm4Mw6pXWEgXOOaNC29rm3JXPEl2MkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhMDKyFJPpHkqiQXTFh2zyTfTPKz7vs9BrV/SaPPOSKNt0EeCTkaePKkZW8CTqmq+wGndPclaSZH4xyRxtbASkhVnQ78etLiZwHHdLePAZ49qP1LGn3OEWm8LR3y/narqo0AVbUxyb1m2jDJocChAHuuHHZMSYuYc0TzctCKNa0jqLNoT0ytqiOram1VrV2+y5LWcSSNIOeItLgNu4RcmWR3gO77VUPev6TR5xyRxsSwS8gJwIu72y8Gvjrk/Usafc4RaUwM8i26nwW+D9w/yaVJXga8B3hikp8BT+zuS9K0nCPSeBvYmVpV9cIZVj1+UPuUNF6cI9J4W7QnpkqSpPFmCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNNCkhSV6b5MIkFyT5bJLtW+SQNLqcI9LoG3oJSbISOAxYW1V/AiwBXjDsHJJGl3NEGg+tXo5ZCtw1yVJgGXB5oxySRpdzRBpxQy8hVXUZ8AHgV8BG4PqqOnnYOSSNLueINB5avBxzD+BZwH2AFcAOSQ6eZrtDk6xLsu7qa28fdkxJi5hzRBoPLV6OeQLwi6q6uqpuBY4HHjl5o6o6sqrWVtXa5bssGXpISYuac0QaAy1KyK+ARyRZliTA44ENDXJIGl3OEWkMtDgn5EzgOGA9cH6X4chh55A0upwj0nhY2mKnVfV24O0t9i1pPDhHpNHnJ6ZKkqQmLCGSJKmJzb4ck+Q50yy+Hji/qq5a+EiSxo1zRNJ0+jkn5GXAfsB3uvuPBX4A7J3kH6rq0wPKJml8OEckTdFPCbkDeGBVXQmQZDfgY8CfAacDDg9Jm+MckTRFP+eErN40ODpXAXtX1a+BWwcTS9KYcY5ImqKfIyFnJDkR+GJ3/78ApyfZAbhuUMEkjRXniKQp+ikhr6I3MPYHAnwK+FJVFfC4AWaTND6cI5Km2GwJ6YbEcd2XJM2Zc0TSdDZ7TkiS5yT5WZLrk9yQ5MYkNwwjnKTx4ByRNJ1+Xo55H/CMqvLiUJK2lHNE0hT9vDvmSgeHpHlyjkiaop8jIeuSfB74CnDzpoVVdfygQkkaO84RSVP0U0J2An4HPGnCsgIcHpL65RyRNEU/7445ZBhBJI0v54ik6cxYQpK8oarel+Qj9P5iuZOqOmygySSNPOeIpNnMdiRk00lk64YRZDY/PW8ZB61Y0zqGpLlbNHNE0uIzYwmpqq91N8+oqouGlEfSGHGOSJpNPyemHp1kJfDv9K52eUZVnT/YWJLGjHNE0hT9nJj66CTbAQ8DHgt8Pcndquqegw4naTw4RyRNZ7MlJMmjgAO6r52BE4EzBhtL0jhxjkiaTj8vx5xG76SydwPfqKpbBhtJ0hhyjkiaop8Ssgu9y28/GjgsyR3A96vq7waaTNI4cY5ImqKfc0KuS3IRsArYA3gksO2gg0kaH84RSdPp55yQ/wB+AnwX+DhwiIdSJc2Fc0TSdPp5OeZ+VXXHwJNIGmfOEUlTbNPHNiuSfDnJVUmuTPKlJHsMPJmkceIckTRFPyXkk8AJwApgJfC1bpkk9cs5ImmKfkrI8qr6ZFXd1n0dDSwfcC5J48U5ImmKfkrINUkOTrKk+zoYuHbQwSSNFeeIpCn6KSEvBZ4PXAFsBJ7bLZOkfjlHJE3Rz+eE/Ap45hCySBpTzhFJ0+nnc0KWAy8HVk/cvqr8K0ZSX5wjkqbTz+eEfJXehaa+Bdw+2DiSxpRzRNIU/ZSQZVX1xoEnkTTOnCOSpujnxNQTkzx1IXeaZOckxyX5cZINSfZbyMeXtOg4RyRNMeORkCQ3AgUEeHOSm4Fbu/tVVTvNY78fBv6tqp6bZDtg2TweS9Ii5RyRNJsZS0hV7TiIHSbZid7lvF/S7ecWwAtZSWPIOSJpNv28O2Z/4Jyquqn7gKF9gQ91b7nbEn8MXA18MslDgLOBV1fVTZP2eyhwKMD2/oEjjbTFMEf2XNnPKXDaGpx0+TkDedyDVqwZyOOOs37OCfkY8Lvuif4G4JfAp+exz6X0BtDHquqhwE3AmyZvVFVHVtXaqlq7LXeZx+4kLQLN58jyXZbMY3eSBqGfEnJbVRXwLODDVfVhYD6HWC8FLq2qM7v7x9EbJpLGl3NE0hT9lJAbkxwOHAx8PckSYNst3WFVXQFckuT+3aLHAz/a0seTNBKcI5Km6KeE/AVwM/Cy7om/Enj/PPf718CxSc4D1gDvmufjSVrcnCOSpujn2jFXAEdMuP8r4FPz2WlVnQOsnc9jSBodzhFJ0+nnSIgkSdKCs4RIkqQmZi0hSZYk+cywwkgaP84RSTOZtYRU1e3A8u4jkSVpzpwjkmbSz0cIXgx8L8kJ9D4QCICqOmLGn5CkO7sY54ikSfopIZd3X9swvw8XkrT1co5ImqKft+i+AyDJDpOvyyBJ/XCOSJrOZt8dk2S/JD8CNnT3H5Lkfw48maSx4RyRNJ1+3qL7IeAg4FqAqjqX3iW0JalfH8I5ImmSvj4npKoumbTo9gFkkTTGnCOSJuvnxNRLkjwSqO4tdofRHVKVpD45RyRN0c+RkFcCr6J3walL6V0o6lUDzCRp/DhHJE3Rz7tjrgFeNIQsksaUc0TSdGYsIUk+AtRM66vqsIEkkjQ2nCOSZjPbkZB1Q0shaVw5RyTNaMYSUlXHDDOIpPHjHJE0m82eE5JkOfBG4EHA9puWV9WBA8wlaYw4R7SYHLRiTesI6vTz7phj6b2V7j7AO+hdiOrfB5hJ0vhxjkiaop8SsktVHQXcWlWnVdVLgUcMOJek8eIckTRFPx9Wdmv3fWOSp9G7EuYeg4skaQw5RyRN0U8JeWeSuwOvAz4C7AS8dqCpJI0b54ikKfr5sLITu5vXA48bbBxJ48g5Imk6mz0nJMneSU5JckF3f58kbx18NEnjwjkiaTr9nJj6v4DD6V7TrarzgBcMMpSkseMckTRFPyVkWVWdNWnZbYMII2lsOUckTdFPCbkmyV50139I8lxg40BTSRo3zhFJU/Tz7phXAUcCD0hyGfAL4OCBppI0bpwjkqbo590xFwFPSLIDsE1V3Tj4WJLGiXNE0nRmLCFJ/maG5QBU1REDyiRpTDhHJM1mtiMhO3bf7w88DDihu/8M4PRBhpI0NpwjkmY0YwmpqncAJDkZ2HfT4dMkfw98cSjpJI0054ik2fTz7pg9gVsm3L8FWD2QNJLGlXNE0hT9vDvm08BZSb5M7+11fw4cM9BUksaNc0TSFP28O+Yfk/wrcEC36JCq+uFgY0kaJ84RSdPp50gIVbUeWL+QO06yBFgHXFZVT1/Ix5a0+DhHJE3Wzzkhg/JqYEPD/Usafc4RaYQ1KSFJ9gCeBvzvFvuXNPqcI9Loa3Uk5EPAG4A7Gu1f0uj7EM4RaaT1dU7IQkrydOCqqjo7yWNn2e5Q4FCA7Vk2nHCSRsKWzJE9Vw593GmROunycwbyuAetWDOQxx1nLY6E7A88M8nFwOeAA5N8ZvJGVXVkVa2tqrXbcpdhZ5S0uM15jizfZcmwM0rajKGXkKo6vKr2qKrVwAuAb1eVV9OU1DfniDQeWr47RpIkbcWavkhaVacCp7bMIGm0OUek0eWREEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE0tbB5AkaZgOWrGmdQR1PBIiSZKasIRIkqQmLCGSJKkJS4gkSWrCEiJJkpqwhEiSpCYsIZIkqQlLiCRJasISIkmSmrCESJKkJiwhkiSpCUuIJElqwhIiSZKasIRIkqQmLCGSJKkJS4gkSWrCEiJJkpqwhEiSpCaGXkKSrErynSQbklyY5NXDziBptDlHpPGwtME+bwNeV1Xrk+wInJ3km1X1owZZJI0m54g0BoZ+JKSqNlbV+u72jcAGYOWwc0gaXc4RaTy0OBLyB0lWAw8Fzpxm3aHAoQB7rlzKSevOGWo2zc9BK9a0jqCtxFzmiKTFpdmJqUnuBnwJeE1V3TB5fVUdWVVrq2rt8l2WDD+gpEXPOSKNtiYlJMm29AbHsVV1fIsMkkabc0QafS3eHRPgKGBDVR0x7P1LGn3OEWk8tDgSsj/w34ADk5zTfT21QQ5Jo8s5Io2BoZ+pVVXfBTLs/UoaH84RaTz4iamSJKkJS4gkSWrCEiJJkpqwhEiSpCYsIZIkqQlLiCRJasISIkmSmrCESJKkJiwhkiSpCUuIJElqwhIiSZKasIRIkqQmLCGSJKkJS4gkSWrCEiJJkpqwhEiSpCZSVa0zbFaSG4GftM7Rp12Ba1qHmINRyjtKWWG08t6/qnZsHWKQnCMDNUp5RykrjFbeOc+RpYNKssB+UlVrW4foR5J1o5IVRivvKGWF0cqbZF3rDEPgHBmQUco7SllhtPJuyRzx5RhJktSEJUSSJDUxKiXkyNYB5mCUssJo5R2lrDBaeUcp65Yapd9xlLLCaOUdpawwWnnnnHUkTkyVJEnjZ1SOhEiSpDFjCZEkSU0suhKS5BNJrkpywYRl70/y4yTnJflykp0bRvyD6bJOWPe3SSrJri2yTWemvEn+OslPklyY5H2t8k00w/+DNUl+kOScJOuSPLxlxk2SrErynSQbun/DV3fL75nkm0l+1n2/R+usMGveRfk82xLOkcFxjgzGVjtHqmpRfQGPBvYFLpiw7EnA0u72e4H3ts45U9Zu+SrgJOCXwK6tc27m3/ZxwLeAu3T379U65yxZTwae0t1+KnBq65xdlt2BfbvbOwI/BR4EvA94U7f8TYvo/+1MeRfl82wB//8syt/POTL0rM6Rwead0/Ns0R0JqarTgV9PWnZyVd3W3f0BsMfQg01juqydDwJvABbVWb8z5P0r4D1VdXO3zVVDDzaNGbIWsFN3++7A5UMNNYOq2lhV67vbNwIbgJXAs4Bjus2OAZ7dJOAkM+VdrM+zLeEcGRznyGBsrXNk0ZWQPrwU+NfWIWaS5JnAZVV1bussfdobOCDJmUlOS/Kw1oFm8Rrg/UkuAT4AHN42zlRJVgMPBc4EdquqjdB7wgL3ahhtWpPyTrSon2cLYFH/fs6RgXoNzpEFNZ85MlIlJMlbgNuAY1tnmU6SZcBbgLe1zjIHS4F7AI8AXg98IUnaRprRXwGvrapVwGuBoxrnuZMkdwO+BLymqm5onWdzZsq72J9n87XYfz/nyMA5RxbQfOfIyJSQJC8Gng68qLoXmxahvYD7AOcmuZjeYaj1Sf6oaarZXQocXz1nAXfQu2DSYvRi4Pju9heBRXFCGUCSbek9EY+tqk0Zr0yye7d+d2BRHKKGGfOOyvNsi43I7+ccGSznyAJZiDkyEiUkyZOBNwLPrKrftc4zk6o6v6ruVVWrq2o1vSfmvlV1ReNos/kKcCBAkr2B7Vi8V2y8HHhMd/tA4GcNs/xB9xffUcCGqjpiwqoT6A08uu9fHXa26cyUd1SeZ1tqVH4/58jAOUcWwILNkdZn2E7+Aj4LbARupffkexnwc+AS4Jzu6+Otc86UddL6i1lcZ7VP92+7HfAZ4AJgPXBg65yzZH0UcDZwLr3XHv+0dc4u66Ponex23oT/o08FdgFOoTfkTgHu2TrrZvIuyufZAv7/WZS/n3Nk6FmdI4PNO6fnmR/bLkmSmhiJl2MkSdL4sYRIkqQmLCGSJKkJS4gkSWrCEiJJkpqwhIgkpyZZ2zqHpNHlHNGWsIRo4JIsaZ1B0mhzjownS8hWIsnqJD9OckyS85Ic112jYvJ2H0uyLsmFSd7RLXt8ki9P2OaJSY7vbj8pyfeTrE/yxe46AiS5OMnbknwXeN6kfTwvyQVJzk1yerfsJUk+OmGbE5M8trv92yTvTXJ2km8leXj3V9dF3YW+JA2Bc0QLzRKydbk/cGRV7QPcAPyPabZ5S1WtBfYBHpNkH+DbwAOTLO+2OQT4ZJJdgbcCT6iqfYF1wN9MeKzfV9Wjqupzk/bxNuCgqnoI0M+Tfwfg1Kr6U+BG4J3AE4E/B/6hj5+XtHCcI1owlpCtyyVV9b3u9mfofezuZM9Psh74IfBg4EHV+1jdTwMHJ9kZ2I/e5ZkfATwI+F6Sc+hd1+DeEx7r8zPk+B5wdJKXA/0cYr0F+Lfu9vnAaVV1a3d7dR8/L2nhOEe0YJa2DqChmvwZ/Xe6n+Q+wN8CD6uq3yQ5Gti+W/1J4GvA74EvVtVt3QWMvllVL5xhfzdNG6LqlUn+DHgacE6SNfQu+TyxFG8/4fat9Z/XF7gDuLl7nDuS+H9YGi7niBaMR0K2Lnsm2a+7/ULgu5PW70TvCX99kt2Ap2xaUVWX07v65FuBo7vFPwD2T3JfgCTLuitozirJXlV1ZlW9jd6VNlfRu0jXmiTbJFnFIrq8tqQ7cY5owdj+ti4bgBcn+Rd6V2T82MSVVXVukh8CFwIX0TvcOdGxwPKq+lG3/dVJXgJ8Nsldum3eCvx0Mznen+R+QOhdFfLcbvkv6B0a3XQlTkmLj3NEC8ar6G4lkqwGTqyqP5nHY3wU+GFVHbVgwSSNDOeIFppHQtSXJGfTO8T6utZZJI0m54gm80iIJElqwhNTJUlSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVIT/x8q4jv+kwX9YQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAexElEQVR4nO3deZhkBXnv8e9vumdhhhkGEQgCOkbFNQQJGhVXXKLGJTGJMVfudYuYXJ+4ZDGaeE3M9caoidEkz9VwQwSVq0bFPQsuEdGrKCIoilsQZBkWQWBgYJjpfu8fdcY0vcxUz5zTp6vm+3mefqbqVPV73i7q/Hjr1Kk6qSokSZKW2oq+G5AkSfsmhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLh5B9UJKbk/z0Lm6/JMnjlrInSePJvNGuOITsg6pq/6q6GCDJqUletxTrTfLcJJ9finVJWh76yhuNBocQSZLUC4eQMZHkeUk+NuP695P804zrlyU5prlcSe6Z5CTg2cArml2mH5tR8pgkX09yY5L3JVkzo9YLm/rXJ/lokrs0yzc1tSdn3PezSX4zyX2BtwMPbdZ1wy7+jouSbElycZIXzbr96UnOT3JTkv9I8sRm+QFJTkmyOckVSV6XZGKPH1BJC1rKvJm13nsk+UyS65L8KMnpSTbOuP3IJGckuba5z9/NuO35Tbb8OMm/Jblbaw+I9phDyPg4C3hEkhVJDgNWAscDNO/H7g98feYvVNXJwOnAG5tdpk+dcfMzgScCdweOBp7b1DoBeH1z+2HApcB7d9dcVV0E/BbwxWZdGxe46zXAU4ANwPOAv05ybLPuBwPvBP4A2Ag8Erik+b3TgB3APYEHAk8AfnN3fUnaI0uSN/MIg/y5C3Bf4EjgT5v1TgAfZ5BJm4DDabIpyS8BfwQ8AzgYOBt4z5796WrT5O7volFQVRcn2QIcAxwF/BuDVxf3AR4KnF1V04so+TdVdSVA84rlmGb5s4F/rKrzmtteBfw4yaaW/o5PzLh6VpIzgUcA5wEvaNb9yeb2K5oeDgWeBGysqluBW5L8NXAS8Pdt9CXpPy1h3sxe7/eB7zdXr03yZuBPmusPZjCc/EFV7WiW7TwG7UXA65sXQyT5c+CPktytqi5dRJ9qmUPIeDkLeDSDvQFnATcAj2IQCmctstZVMy5vZbBx0/x73s4bqurmJNcxeNVxxZ40PVOSJzEIlaMY7KlbC3yjuflI4J/n+bW7MXgltjnJzmUrgMv2th9JC1qKvLmDJIcAf8Pghcl6Btv5j5ubjwQunTGAzHQ34K1J/mpmOQa55RDSI9+OGS87Q+ERzeWzGITCo1g4FBZ7GuUrGWzQACRZBxzEYAC5pVm8dsb9f2rYdSVZDXwQ+Evg0OYtm39mEBYwGCruMc+vXgZsA+5cVRubnw1Vdf9h/yhJi7YUeTPb65saR1fVBuBE7pgPd515TNoMlwEvmpEPG6tqv6r6f3vZj/aSQ8h4OQt4DLBfVV3O4H3PJzIYEr62wO9cDSz4Gf55/F/geUmOaYaGPwfOqapLqupaBsPIiUkmkjyfOw4NVwNHJFm1QO1VwGrgWmBHs1fkCTNuP6VZ92Ob96IPT3KfqtoMnAn8VZINzW33SPKoRfxdkhZnKfJmtvXAzcANSQ5ncHzYTl8GNgN/kWRdkjVJjm9uezvwqiT3h58cyP5re9GHWuIQMkaq6rsMNtCzm+s3ARcDX6iqqQV+7RTgfkluSPLhIdbxaeB/MNhjsZnBkPGsGXd5IYNguA64PzDzlcZngG8CVyX50Ty1twAvAf6JwS7W/wJ8dMbtX6Y5WBW4kUEI7twr898YDDHfan73AwwOnJXUgaXIm3m8FjiWwfb/CeCMGf1MAU9l8PbQD4HLgV9vbvsQ8AbgvUluAi5kcByZepaqvd07JkmStHjuCZEkSb1wCJEkSb1wCJEkSb1wCJEkSb0YiS8rW7VyXa1ZtbH1uunqoNzpxXxR4JC6On64q8dgspvTtkyt6aju6vZrTqzf3n5R4KfXXNd6zcsvn+L666ez+3uOrsk162rV+ju1XndiezfbUK1o/z/Him0dZBPQWUB18BgATE92VHdl+3Wzcb7vXtt799yv/Ry57LIdi86RkRhC1qzayEMe8KLd33GRsq2b/0nk1tvbr7m9myciHdWdPmhDJ3Vvus/Gbupuan+n4IYTrtr9nfbA++73ztZrPuXJcz4xPXZWrb8T93nGy1uvu/+V3WxDO/Zr/zm5/6VbW68JwI5uhpup/Rf6SqG9s+2glZ3U3XKX9v+XuuapV7deE+AjD2g/R35hD3LEt2MkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvOhtCkvxjkmuSXDhj2Z2SfDLJ95p/D+xq/ZJGnzkijbcu94ScCjxx1rJXAp+uqnsBn26uS9JCTsUckcZWZ0NIVX0OuH7W4qcDpzWXTwN+qav1Sxp95og03iaXeH2HVtVmgKranOSQhe6Y5CTgJIDVazayfcOq1puZ3NJ6SQAyMdF6zRW372i9JgDbbu+kbK1s/zEA2L42ndTdevStrdf8n/c4s/WaAEdM7t96zVX5ces1O7RHOTJx0EauP7795/vUV1a3XhNguv3IY8X2/dovCmSqk7Ksvn5bJ3WnJ7vJkRuO2d56zX846sOt1wQ4ZGJd6zVXznm9sHvL9sDUqjq5qo6rquNWrmz/wZI0/mbmyMR6c0RabpZ6CLk6yWEAzb/XLPH6JY0+c0QaE0s9hHwUeE5z+TnAR5Z4/ZJGnzkijYkuP6L7HuCLwL2TXJ7kBcBfAI9P8j3g8c11SZqXOSKNt84OTK2q31jgpsd2tU5J48Uckcbbsj0wVZIkjTeHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1IvJvhsYygrYsXai9bJTa9a0XhNgzTW3tV5zx9r9Wq8JQHXzGNREN/Ptj+/XSVkefc/vtV5z/YpbW68JcPN0+8+vKaZbr7ncTGwNB5y7uvW6W+7ezWO3+rr2t6FtB7SfowC3HZRO6u53TTc5ct3PdNPv7zzsk63XXJfbW68JsK3af95OU4v+HfeESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXvQyhCR5eZJvJrkwyXuSrOmjD0mjyxyRRt+SDyFJDgdeAhxXVQ8AJoBnLXUfkkaXOSKNh77ejpkE9ksyCawFruypD0mjyxyRRtzkUq+wqq5I8pfAD4FbgTOr6sxd/c7UqrDl8InWe1l39XTrNQG2r1/Ves0V27vpdeK2qU7qXnP02k7q3vXYKzqpe+jqm1qvecv06tZrNpU7qjs69iRHpvef5paH39x+L1d281y//cBqveb1G1svCcCBF7XfK8CPju2kLEf8zOZO6m6Zav8dwSt2HNh6TYAHrr6uk7qL1cfbMQcCTwfuDtwFWJfkxHnud1KSc5Ocu+NWQ1fSf9qTHJm6yRyRlps+3o55HPCDqrq2qrYDZwAPm32nqjq5qo6rquMm91u35E1KWtYWnSMTG8wRabnpYwj5IfCQJGuTBHgscFEPfUgaXeaINAaWfAipqnOADwDnAd9oejh5qfuQNLrMEWk8LPmBqQBV9SfAn/SxbknjwRyRRp/fmCpJknrhECJJknqx27djkjxjnsU3At+oqmvab0nSuDFHJM1nmGNCXgA8FPj35vqjgS8BRyX5s6p6V0e9SRof5oikOYYZQqaB+1bV1QBJDgXeBvw88DnA8JC0O+aIpDmGOSZk087gaFwDHFVV1wPbu2lL0pgxRyTNMcyekLOTfBx4f3P9V4DPJVkH3NBVY5LGijkiaY5hhpAXMwiM44EA7wQ+WFUFPKbD3iSND3NE0hy7HUKakPhA8yNJi2aOSJrPbo8JSfKMJN9LcmOSm5JsSdL+ec8ljS1zRNJ8hnk75o3AU6vKk0NJ2lPmiKQ5hvl0zNUGh6S9ZI5ImmOYPSHnJnkf8GFg286FVXVGV01JGjvmiKQ5hhlCNgBbgSfMWFaA4SFpWOaIpDmG+XTM85aiEUnjyxyRNJ8Fh5Akr6iqNyb5WwavWO6gql7SaWeSRp45ImlXdrUnZOdBZOcuRSO7VJDp9sveeudhjstdvFsOXdV6zYltu7/PntixNp3U3b6+k7JcfNnBndQ9cPXW1mseturG1msCsN/13dTtxrLJkdyygjVf3r/1ujv2a70kALdvnDOz7bU113WzvV/78zs6qbv6mmGOGFi8Sy/tJke2HnxJ6zUv236n1msCbJ2+qvWa03NfZ+zWgv+Fq+pjzcWzq+riPW1K0r7LHJG0K8OMmacmORz4CoOzXZ5dVd/oti1JY8YckTTHMAemPjLJKuBBwKOBTyTZv6q62UckaeyYI5Lms9shJMnDgUc0PxuBjwNnd9uWpHFijkiazzBvx5zF4KCy1wP/XFW3d9uSpDFkjkiaY5gh5CAGp99+JPCSJNPAF6vqf3TamaRxYo5ImmOYY0JuSHIxcCRwBPAwYGXXjUkaH+aIpPkMc0zIfwDfAT4PvB14nrtSJS2GOSJpPsO8HXOvqurgq8Ik7UPMEUlzDPOVoXdJ8qEk1yS5OskHkxzReWeSxok5ImmOYYaQdwAfBe4CHA58rFkmScMyRyTNMcwQcnBVvaOqdjQ/pwLdfPG+pHFljkiaY5gh5EdJTkwy0fycCFzXdWOSxoo5ImmOYYaQ5wPPBK4CNgO/2iyTpGGZI5LmGOZ7Qn4IPG0JepE0pswRSfMZ5ntCDgZeCGyaef+q8lWMpKGYI5LmM8z3hHyEwYmmPgVMdduOpDFljkiaY5ghZG1V/WHnnUgaZ+aIpDmGOTD140me3OZKk2xM8oEk305yUZKHtllf0rJjjkiaY8E9IUm2AAUE+KMk24DtzfWqqg17sd63Av9aVb+aZBWwdi9qSVqmzBFJu7LgEFJV67tYYZINDE7n/dxmPbcDnshKGkPmiKRdGebTMccD51fVLc0XDB0LvKX5yN2e+GngWuAdSX4W+Crw0qq6ZdZ6TwJOAli17kAmb93Dte1CTbRfE2BqZVqvObG9m3N/5eZOyjK1uv3HAIAdw7yDuHiXb9nYes3bDhzmkKvFu356R+s1p6parznTcsiRlesPHOx/adnE9vZrAkytb//43a1ru9ku12zu5rm+ov2nOgCrrumm309dflTrNX/lbhe0XhNgim63+WENk+hvA7Y2G/orgEuBd+3FOicZBNDbquqBwC3AK2ffqapOrqrjquq4yTXr9mJ1kpaB3nNkYq05Ii03wwwhO6qqgKcDb62qtwJ7s4v1cuDyqjqnuf4BBmEiaXyZI5LmGGYI2ZLkVcCJwCeSTAAr93SFVXUVcFmSezeLHgt8a0/rSRoJ5oikOYYZQn4d2Aa8oNnwDwfetJfr/R3g9CRfB44B/nwv60la3swRSXMMc+6Yq4A3z7j+Q+Cde7PSqjofOG5vakgaHeaIpPl081EDSZKk3XAIkSRJvdjlEJJkIsm7l6oZSePHHJG0kF0OIVU1BRzcfCWyJC2aOSJpIcN8bdwlwBeSfJTBFwIBUFVvXvA3JOmOLsEckTTLMEPIlc3PCvbuy4Uk7bvMEUlzDPMR3dcCJFk3+7wMkjQMc0TSfHb76ZgkD03yLeCi5vrPJvnfnXcmaWyYI5LmM8xHdN8C/AJwHUBVXcDgFNqSNKy3YI5ImmWo7wmpqstmLWr/HNOSxpo5Imm2YQ5MvSzJw4BqPmL3EppdqpI0JHNE0hzD7An5LeDFDE44dTmDE0W9uMOeJI0fc0TSHMN8OuZHwLOXoBdJY8ockTSfBYeQJH8L1EK3V9VLOulI0tgwRyTtyq72hJy7ZF1IGlfmiKQFLTiEVNVpS9mIpPFjjkjald0eE5LkYOAPgfsBa3Yur6oTOuzrjgoyveAe3T2XtF8TWLWl/V5X7Gi9JNDR4wqs7Oo7Mae6+W+WtP84bJ1a3XrNUbUccmR6JWz9qfb/O2+4uPWSAKzZPMyHFxcn062XBGDF9m7qdiUdfTh81WT7he80eXPrNZeTYT4dczqDj9LdHXgtgxNRfaXDniSNH3NE0hzDDCEHVdUpwPaqOquqng88pOO+JI0Xc0TSHMPs79u5o21zkl9kcCbMI7prSdIYMkckzTHMEPK6JAcAvwf8LbABeHmnXUkaN+aIpDmG+bKyjzcXbwQe0207ksaROSJpPrs9JiTJUUk+neTC5vrRSV7dfWuSxoU5Imk+wxyY+n+AV9G8p1tVXwee1WVTksaOOSJpjmGGkLVV9eVZyzr61gpJY8ockTTHMEPIj5Lcg+b8D0l+FdjcaVeSxo05ImmOYT4d82LgZOA+Sa4AfgCc2GlXksaNOSJpjmE+HXMx8Lgk64AVVbWl+7YkjRNzRNJ8FhxCkvzuAssBqKo3d9STpDFhjkjalV3tCVnf/Htv4EHAR5vrTwU+12VTksaGOSJpQQsOIVX1WoAkZwLH7tx9muRPgfcvSXeSRpo5ImlXhvl0zF2B22dcvx3Y1Ek3ksaVOSJpjmE+HfMu4MtJPsTg43W/DJzWaVeSxo05ImmOYT4d87+S/AvwiGbR86rqa922JWmcmCOS5jPMnhCq6jzgvDZXnGQCOBe4oqqe0mZtScuPOSJptmGOCenKS4GLely/pNFnjkgjrJchJMkRwC8C/9DH+iWNPnNEGn197Ql5C/AKYLqn9UsafW/BHJFG2lDHhLQpyVOAa6rqq0kevYv7nQScBLBq7YGs6OB8mxNbu8mu6ZVpv2hV+zWBTHVSlkoHjwEwcUs3c/PUdPt1t9dE6zU1sCc5MrnhQFbd0P7z8tZDWi8JwPb929/mN3679ZIArOgoR24+vJsc2b6+mzytar/fb996WOs1AabXf7f1mnvyqPaxJ+R44GlJLgHeC5yQ5N2z71RVJ1fVcVV13MrV65a6R0nL26JzZGKdOSItN0s+hFTVq6rqiKraBDwL+ExVeTZNSUMzR6Tx0OenYyRJ0j5syY8JmamqPgt8ts8eJI02c0QaXe4JkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvZjsu4GhFExsr9bLTt463XpNgOkdab3m1Kr2awJMbu3mMdi+rqP5drqbx+HW21d2UrcLHWwKdFBy2alVxa2btrded91/dPPcWXdl+zVrov2aALm9m7oH/KCbfPrxqm7y6ZZtq1qvecjKLa3XBLit2t/qaw9quidEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1YsmHkCRHJvn3JBcl+WaSly51D5JGmzkijYfJHta5A/i9qjovyXrgq0k+WVXf6qEXSaPJHJHGwJLvCamqzVV1XnN5C3ARcPhS9yFpdJkj0njoY0/ITyTZBDwQOGee204CTgJYM7GeDedc1n4DU1Pt1wRq2+0dFJ1uvyYwdcONndTdb82aTuoeePS9Oql7/QMOaL3me37m+NZrAqx9bPvPr+unPt96zaUydI6sOoB7n3xb6+uviW2t1wSYnmz/NeLK625pvSbA1EXf66Tu5N3v1kndA78y0Undqx9zaOs1333IY1uvCTD9zLRe87qpsxf9O70dmJpkf+CDwMuq6qbZt1fVyVV1XFUdt2rFfkvfoKRlbzE5snJy3dI3KGmXehlCkqxkEBynV9UZffQgabSZI9Lo6+PTMQFOAS6qqjcv9foljT5zRBoPfewJOR74r8AJSc5vfp7cQx+SRpc5Io2BJT8wtao+D7R/RIykfYY5Io0HvzFVkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1IlXVdw+7lWQL8J2++xjSnYEf9d3EIoxSv6PUK4xWv/euqvV9N9Elc6RTo9TvKPUKo9XvonNksqtOWvadqjqu7yaGkeTcUekVRqvfUeoVRqvfJOf23cMSMEc6Mkr9jlKvMFr97kmO+HaMJEnqhUOIJEnqxagMISf33cAijFKvMFr9jlKvMFr9jlKve2qU/sZR6hVGq99R6hVGq99F9zoSB6ZKkqTxMyp7QiRJ0phxCJEkSb1YdkNIkn9Mck2SC2cse1OSbyf5epIPJdnYY4s/MV+vM277/SSV5M599DafhfpN8jtJvpPkm0ne2Fd/My3wPDgmyZeSnJ/k3CQP7rPHnZIcmeTfk1zUPIYvbZbfKcknk3yv+ffAvnuFXfa7LLezPWGOdMcc6cY+myNVtax+gEcCxwIXzlj2BGCyufwG4A1997lQr83yI4F/Ay4F7tx3n7t5bB8DfApY3Vw/pO8+d9HrmcCTmstPBj7bd59NL4cBxzaX1wPfBe4HvBF4ZbP8lcvoebtQv8tyO2vx+bMs/z5zZMl7NUe67XdR29my2xNSVZ8Drp+17Myq2tFc/RJwxJI3No/5em38NfAKYFkd9btAv78N/EVVbWvuc82SNzaPBXotYENz+QDgyiVtagFVtbmqzmsubwEuAg4Hng6c1tztNOCXemlwloX6Xa7b2Z4wR7pjjnRjX82RZTeEDOH5wL/03cRCkjwNuKKqLui7lyEdBTwiyTlJzkryoL4b2oWXAW9Kchnwl8Cr+m1nriSbgAcC5wCHVtVmGGywwCE9tjavWf3OtKy3sxYs67/PHOnUyzBHWrU3OTJSQ0iSPwZ2AKf33ct8kqwF/hh4Td+9LMIkcCDwEOAPgH9Kkn5bWtBvAy+vqiOBlwOn9NzPHSTZH/gg8LKquqnvfnZnoX6X+3a2t5b732eOdM4cadHe5sjIDCFJngM8BXh2NW82LUP3AO4OXJDkEga7oc5L8lO9drVrlwNn1MCXgWkGJ0xajp4DnNFcfj+wLA4oA0iyksGGeHpV7ezx6iSHNbcfBiyLXdSwYL+jsp3tsRH5+8yRbpkjLWkjR0ZiCEnyROAPgadV1da++1lIVX2jqg6pqk1VtYnBhnlsVV3Vc2u78mHgBIAkRwGrWL5nbLwSeFRz+QTgez328hPNK75TgIuq6s0zbvoog8Cj+fcjS93bfBbqd1S2sz01Kn+fOdI5c6QFreVI30fYzv4B3gNsBrYz2PheAHwfuAw4v/l5e999LtTrrNsvYXkd1T7fY7sKeDdwIXAecELffe6i14cDXwUuYPDe48/13WfT68MZHOz29RnP0ScDBwGfZhBynwbu1Hevu+l3WW5nLT5/luXfZ44sea/mSLf9Lmo782vbJUlSL0bi7RhJkjR+HEIkSVIvHEIkSVIvHEIkSVIvHEIkSVIvHEJEks8mOa7vPiSNLnNEe8IhRJ1LMtF3D5JGmzkynhxC9hFJNiX5dpLTknw9yQeac1TMvt/bkpyb5JtJXtsse2ySD824z+OTnNFcfkKSLyY5L8n7m/MIkOSSJK9J8nng12at49eSXJjkgiSfa5Y9N8nfzbjPx5M8url8c5I3JPlqkk8leXDzquvi5kRfkpaAOaK2OYTsW+4NnFxVRwM3Af99nvv8cVUdBxwNPCrJ0cBngPsmObi5z/OAdyS5M/Bq4HFVdSxwLvC7M2rdVlUPr6r3zlrHa4BfqKqfBYbZ+NcBn62qnwO2AK8DHg/8MvBnQ/y+pPaYI2qNQ8i+5bKq+kJz+d0MvnZ3tmcmOQ/4GnB/4H41+FrddwEnJtkIPJTB6ZkfAtwP+EKS8xmc1+BuM2q9b4E+vgCcmuSFwDC7WG8H/rW5/A3grKra3lzeNMTvS2qPOaLWTPbdgJbU7O/ov8P1JHcHfh94UFX9OMmpwJrm5ncAHwNuA95fVTuaExh9sqp+Y4H13TJvE1W/leTngV8Ezk9yDINTPs8citfMuLy9/vP8AtPAtqbOdBKfw9LSMkfUGveE7FvumuShzeXfAD4/6/YNDDb4G5McCjxp5w1VdSWDs0++Gji1Wfwl4Pgk9wRIsrY5g+YuJblHVZ1TVa9hcKbNIxmcpOuYJCuSHMkyOr22pDswR9Qap799y0XAc5L8PYMzMr5t5o1VdUGSrwHfBC5msLtzptOBg6vqW839r03yXOA9SVY393k18N3d9PGmJPcCwuCskBc0y3/AYNfozjNxSlp+zBG1xrPo7iOSbAI+XlUP2Isafwd8rapOaa0xSSPDHFHb3BOioST5KoNdrL/Xdy+SRpM5otncEyJJknrhgamSJKkXDiGSJKkXDiGSJKkXDiGSJKkXDiGSJKkX/x+l82rEry02wgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(policy.argmax(-1))\n",
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23:20:32 [INFO] average episode reward = 0.04 ± 0.95\n"
     ]
    }
   ],
   "source": [
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_policy(env, policy)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
